{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60c8f7fb",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:28.092386Z",
     "iopub.status.busy": "2023-02-19T19:32:28.091958Z",
     "iopub.status.idle": "2023-02-19T19:32:29.586713Z",
     "shell.execute_reply": "2023-02-19T19:32:29.585658Z"
    },
    "papermill": {
     "duration": 1.503379,
     "end_time": "2023-02-19T19:32:29.589440",
     "exception": false,
     "start_time": "2023-02-19T19:32:28.086061",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from xgboost import XGBRFClassifier\n",
    "\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.metrics import make_scorer, fbeta_score\n",
    "from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c91bf783",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:29.599078Z",
     "iopub.status.busy": "2023-02-19T19:32:29.598725Z",
     "iopub.status.idle": "2023-02-19T19:32:29.610020Z",
     "shell.execute_reply": "2023-02-19T19:32:29.608914Z"
    },
    "papermill": {
     "duration": 0.01999,
     "end_time": "2023-02-19T19:32:29.613144",
     "exception": false,
     "start_time": "2023-02-19T19:32:29.593154",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### DEFINE THRESHOLD CLASSIFIER AS SKLEARN COMPATIBLE ESTIMATOR ###\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.base import clone, BaseEstimator, ClassifierMixin\n",
    "\n",
    "\n",
    "class ThresholdClassifier(BaseEstimator, ClassifierMixin):\n",
    "    \n",
    "    def __init__(self, estimator, refit=True, val_size=0.3):\n",
    "        self.estimator = estimator\n",
    "        self.refit = refit\n",
    "        self.val_size = val_size\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \n",
    "        def scoring(th, y, prob):\n",
    "            pred = (prob > th).astype(int)\n",
    "            return 0 if not pred.any() else \\\n",
    "                -fbeta_score(y, pred, beta=0.1) \n",
    "        \n",
    "        X_train, X_val, y_train, y_val = train_test_split(\n",
    "            X, y, stratify=y, test_size=self.val_size, \n",
    "            shuffle=True, random_state=1234\n",
    "        )\n",
    "        \n",
    "        self.estimator_ = clone(self.estimator)\n",
    "        self.estimator_.fit(X_train, y_train)\n",
    "        \n",
    "        prob_val = self.estimator_.predict_proba(X_val)[:,1]\n",
    "        thresholds = np.linspace(0,1, 200)[1:-1]\n",
    "        scores = [scoring(th, y_val, prob_val) \n",
    "                    for th in thresholds]\n",
    "        self.score_ = np.min(scores)\n",
    "        self.th_ = thresholds[np.argmin(scores)]\n",
    "        \n",
    "        if self.refit:\n",
    "            self.estimator_.fit(X, y)\n",
    "        if hasattr(self.estimator_, 'classes_'):\n",
    "            self.classes_ = self.estimator_.classes_\n",
    "            \n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        proba = self.estimator_.predict_proba(X)[:,1]\n",
    "        return (proba > self.th_).astype(int)\n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "        return self.estimator_.predict_proba(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0ccfd06c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:29.621970Z",
     "iopub.status.busy": "2023-02-19T19:32:29.621573Z",
     "iopub.status.idle": "2023-02-19T19:32:29.641656Z",
     "shell.execute_reply": "2023-02-19T19:32:29.640808Z"
    },
    "papermill": {
     "duration": 0.027328,
     "end_time": "2023-02-19T19:32:29.644104",
     "exception": false,
     "start_time": "2023-02-19T19:32:29.616776",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### SIMULATE DATA ###\n",
    "\n",
    "X, y = make_classification(\n",
    "    n_samples=5_000, \n",
    "    class_sep=0.5,\n",
    "    n_features=30, n_informative=11, n_redundant=6,\n",
    "    n_classes=2, weights=[0.9, 0.1],  \n",
    "    random_state=1234, shuffle=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "84af1448",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:29.654419Z",
     "iopub.status.busy": "2023-02-19T19:32:29.654017Z",
     "iopub.status.idle": "2023-02-19T19:32:29.808778Z",
     "shell.execute_reply": "2023-02-19T19:32:29.807935Z"
    },
    "papermill": {
     "duration": 0.162333,
     "end_time": "2023-02-19T19:32:29.810836",
     "exception": false,
     "start_time": "2023-02-19T19:32:29.648503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEeCAYAAADb+JPQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbJ0lEQVR4nO3deZxdVZnu8d+biYAkhCEgIdKgMsggM8oYmxYFsaHFlm5QRKYWlQtebL3aitvdYiOjtBgcQARbxabVKCAyiEwRuIwBZEbGGJBJQuaQ5O0/9okckqpTp6rOPu9e+zzfz2d/KnVSSR409WStfdZey9wdEZHUjIgOICIyFCovEUmSyktEkqTyEpEkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJKi8RSZLKS0SSpPISkSSpvEQkSSovEUmSyktEkqTyEpEkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJKi8RSZLKS0SSpPISkSSpvEQkSSovEUmSyktEkqTyEpEkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJo6IDSH1YbgasA0zq45oIjKb4Ozey6eOKP14AvAC82OJ6zjN/oVv/XVJN5u7RGSQxltuGwNZN18YUBbU+MKZLMV4GHgEeblwPAPcCj3jmS7uUQQKpvKQly20jYKemaztgjchMA1gA3A/cDdwMXOuZ/zE2kpRB5SWvY7mtA7wX2BfYG1g3NlFHPAlc27h+55nPDM4jHaDy6nGW2wiKEdW+jWtH6v9GzqPA7yjK7GrP/MXgPDIEKq8eZLmtDuwP7Ae8h+Ime696FbgK+AnwK898XnAeaZPKq4dYbnsARwAfAt4QHKeK5gGXUBTZlZ75q8F5pAWVV81ZbpOAw4DDgU2C46TkReBnFEV2o2f6RqkalVcNWW6jKaaFR1DcfB8Zmyh5DwNnAhd65gujw0hB5VUjltt44FjgeOrxLmHVPA9MBaZqkWw8lVcNWG5rUhTWccCawXF6wQLgAuBMz/zR4Cw9S+WVMMttInAC8ClgXHCcXrQM+CVwmmd+S3CWnqPySpDl9kbgs8AxwGrBcaRwBfA5z/ze6CC9QuWVEMttDeDLwCeBscFxZGXLgB8CJ2oVf/lUXglorII/Evgaxe4MUm0LgDOAkz3z+dFh6krlVXGW267A2cD20Vlk0J6mmEr+NDpIHam8KqrxDuKpFCMuC44jw3MD8CnP/A/RQepE5VVBltuhFNMOTRHrYxFwInCGZ74sOkwdqLwqxHJbn2L90HuCo0h5bgQO88wfjw6SurpvfZIMy+19FBvoqbjqbQ/gHsvt6OggqdPIK5jlNgY4hWKFvO5t9ZZfA0d55s9GB0mRyiuQ5bYJ8FP0TmIvexE4xjP/WXSQ1Ki8glhuhwHfAlaPziKV8G3gOM98SXSQVKi8usxyG0fxF/XD0Vmkcq4F/tEzfyk6SApUXl1kuU0GLqc4LkykL38E9vfM748OUnV6t7FLLLdtgFtQcUlrbwFuttz2iw5SdSqvLrDc3kuxvmeD6CyShPHAJZbbZ6ODVJmmjSWz3I4EvkNxpL3IYP0Q+BfPfFF0kKpReZXIcjsJ+GJ0DkneDcB+nvnc6CBVovIqQWPh6feBj0Rnkdr4PbCvZz4nOkhVqLw6zHJbBfgVxak9Ip10M7CPZ/5KdJAq0A37DrLcRgEXo+KScuwCXN3YUbfnqbw6xHIbCfyY4rxEkbLsDPy2sd9bT1N5dYDlZhT3uA6KziI9YUeKAlsrOkgklVdnTAUOiw4hPWV74BrLbe3oIFFUXsNkuZ0BfCI6h/SkbYFfW26rRgeJoPIaBsvtqxSHvopEeQfwk8YJUz2l5/6DO8Vy+yTwpegcIsA/AGcFZ+g6rfMaAsvt7yhOSNYjP1IlJ3jm34gO0S0qr0Gy3N4K3Ar0/FvVUjlLgfd75ldEB+kGldcgNBYH3gJsHp1FpB+zgXd65g9GBymb7nm1qbGW64eouKTa1qDYTqf2MwOVV/u+gFbPSxo2AX4UHaJsmja2wXJ7N3AlKntJy7Ge+dToEGVReQ3AcptEcRjsOtFZRAZpAbCDZ/5AdJAyaCQxsPNQcUmaVqVYwDomOkgZVF4tWG6HA/tG5xAZhm2Br0WHKIOmjf1oHFP2B4p3b0RS5sC7PfPfRQfpJI28+ncuKi6pBwMurNsWOiqvPlhuRwD7ROcQ6aDJwHejQ3SSpo0r0HRRau5Az3xadIhO0MhrZZouSp2dYbmNjQ7RCSqvJpbbYWi6KPW2MfCv0SE6QdPGBsttNeBRYP3oLCIlmw9s5pnPjA4yHBp5veYzqLikN6wGnBodYrg08gIst3UpRl3jorOIdNEenvn06BBDpZFXIUPFJb3nmynvfZ9s8E6x3DYF/iU6h0iA7YCjokMMVc+XF3Ay2oteetdJjTerktPT5WW57QIcGJ1DJNBEEh199XR5AadFBxCpgM9YbqOjQwxWz5aX5fa3wG7ROUQqYEPgkOgQg9Wz5QV8OjqASIV8rnHITDJ6srwst7cA74/OIVIhWwAHRIcYjAHfZTOzicDRwEbNX+/uR5QXq3TH0aPFLdLC54FfRodo14Ar7M3sJuBG4A6KE3kBcPeflxutHJbbeGAmWpQq0pe9PPNro0O0o531Tau5+/8rPUkbzGwf4D+BkcB57v71Ifw2R6HiEunP54EkyqudqdNlZva+0pMMwMxGAlMpDsTYAjjYzLYY1O+R20jg/5QQT6Qu3tN46qTy2imv4ykKbKGZzWlcr5QdrA87A4+6+2Puvhj4KYO/wXgAxb07EelfEvezBywvdx/n7iPcfWzjx+PcfXw3wq1gA+Dpps9nNl4bjOM6F0ektj7amKVUWlvP9JnZ/sCejU+vc/fLyotUDsttY2BKdA6RBKxPcXum0t/nA468zOzrFFPH+xvX8WZ2ctnB+vAn4E1Nn09uvNau5FYQiwQ6PDrAQNpZKnEPsK27L2t8PhK4y93f3oV8zTlGAQ8Df0dRWrcBh7j7fW39+tzuo7jRLyIDWwSs55nPjg7Sn3YXak5o+nHIyTruvgQ4FrgSeAC4eBDFtS0qLpHBWAX4YHSIVtq553UycJeZXUtx8u6eFGtBus7dLwcuH8Iv/adOZxHpAYcA50eH6E9be9ib2frATo1Pb3X3Z0tN1WGW28PAJtE5RBKzDJjsmT8THaQv/U4bzWzzxsftKd59mNm4JjVeS4LlthUqLpGhGAHsFx2iP62mjSdQ7O1+Rh8/58BepSTqvErP20Uqbm/gvOgQfWnn3cax7r5woNeqynKbAWwTnUMkUS8C63pWrDaoknbebbypzdcqx3KbiIpLZDjWpjhlqHL6nTaa2RspHr9Z1cy2o3inEWA8xYm7Kdg9OoBIDexNsSVWpbS65/Ve4GMUK9nPbHp9DvBvJWbqpD0H/hIRGcDewFC2nypVO/e8PpjwxoN3AMm8MypSUYuAtTzz+dFBmrWzSHUrM9tyxRfd/d9LyNMxjR1Tdb9LZPhWoZjFXBEdpFk7N+znAvMa11KKp803KjFTp+xKseOqiAzf3tEBVjTgyMvdX7fOy8xOp3i+sOp0v0ukc/aIDrCioZygsxrFTfyqU3mJdM6WllulTtxq5+izeylW1EMxDZsIVP1+11heexZTRIZvNeAtwCPRQZZr54Z98+GsS4A/N7anqbKtgDHRIURq5u1UqLza2cP+SYpVtgcABwJblx2qAzaLDiBSQ5X63m9nG+gvAxdSFNg6wAVm9qWygw2Tykuk87q6e/JA2pk2fhjYZvmD2I097WcAJ5WYa7iSOHdOJDFpjbyAWcDYps9XYXAHX0TQyEuk895sub0hOsRyrR7MPpviXcbZwH1mdnXj872BW7sTb8i0+aBI540AtqQi3/+tpo23Nz7eAUxrev260tJ0gOW2AVCZfx1EamYrql5e7n5hN4N0kKaMIuWZFB1guVbTxovd/aAVFqn+VbfPbRwE3awXKc960QGWazVtPL7x8f0tvqaK3hIdQKTG1o0OsFyraeMzjdOxL3D3v+1ipuFaJzqASI1VZuTVcqmEuy8FlplZyCnZQ5RSVpHUVKa82lmkOhe4t7FUYt7yF939uNJSDc+E6AAiNVb9aWOTXzSuZgMfsx1HIy+R8qxpuY32zF+NDtJOeU1w9/9sfsHMju/viytgQnQAkRozitFX+FM27TwedFgfr32swzk6aUJ0AJGaq8TUsdU6r4OBQ4CNzeySpp8aD7xUdrBhGB8dQKTmJkQHgNbTxpuAZyiWHjTvYz8HuKfMUENlua1Oe1NhERm6SnyPtVrn9STwpJm9G1jg7svMbFNgc+DebgUcJN2sFylfJU7lauee1w3AWDPbALgKOBS4oMxQw7BKdACRHlDtkVcTc/f5ZnYkcI67n2pmM0rONVRLowPIII1iIasxOzqGDEJFFkq1VV5mtgvFjqpHNl6rxLCxDyqv1Dgj+CAPsaGOqktIJb7/25k2fhr4AjDN3e8zszcD15aaauhUXqlZyhh+wJ5cwU04c6LjSFsq8X1m7hUZA3aA5TYReC46hwzRRB7naJYwWjvhVty+jl8RHaLVOq+z3P3TZnYpfe/ntX+pyYZmYXQAGYbn2ZhTWcARTGd9do+OI/2qxMir1T2v/2p8PL0bQTpkfnQAGaYlrMr32J09mc672AFj1ehIspJF0QGgzWmjmU0EcPfnS080TJbbQrRkoh4m8QiHM4pRbBwdRV5nC8cfiA7R8oa9mX3FzF4AHgIeNrPnG4fQVtm8gb9EkjCLTTiNdXiBm6KjyOtUYhDTb3mZ2QnAbsBO7r6Wu68JvAPYzcz+b7cCDsHc6ADSQYsZx1R25WZuwFkcHUdYRkWebW418joUONjdH1/+grs/BnwE+GjZwYbhmegAUoKr2JMLeJSlzIyO0uNecnxZdAhoXV6j3f2FFV9s3PcaXV6kYXs6OoCU5Cm24AxWZ3Y1zg3sUZWYMkLr8mo1RK/y8P2p6ABSogVM4Cx24k6ux1kSHacHVaa8Wi2V2MbMXunjdQPGlpSnE1Re9WdcyhQe4B4OZiIjWD86UA+pTHn1O/Jy95HuPr6Pa5y7V3naqPLqFY/yds5kNHO5IzpKD6l+eSVM97x6yTzW4Uy24z6ux6nEjeSaq8z3Vx3LSyOvXuOM4GdM4WfMYFl1RgY1Fb44dblaPZi9nOW2gGrfl5OyjOdZjuHPrMo20VFqanPHH4oOAW2MvMzslHZeqxitBepVr/BGTmdLHuF6vCrb5tXGYuDR6BDLtTNt3LuP1/btdJAOq8S/DBJkGaP4CVO4hNtwXo6OUyMPO16JHSWg9eNBnzCze4HNzOyeputxKnp6UJPbowNIBcxgZ85mLou4PzpKTVTqf8dW67x+AvwGOBn4fNPrc9y9Es82tXBbdACpiL8wmdNYzKHcwN9oq+lhqlR5tVrnNdvdn3D3g4E3AXs1jkMbYWZV36JEIy95zVLGcAF7cgU3a6vpYalUeQ34bqOZZcCOwGbuvqmZTQL+x91360bAobLcnqIoXZHXTOQJjmYxo9k0OkqCtnS8MgXWzg37DwD709gny91nAePKDNUhmjrKyp5nI07lTTzD9OgoiXkZeDA6RLN2ymuxF8MzBzCzN5QbqWNUXtK35VtN/47puLYOb9P0qmyFs1w75XWxmX0XmGBmRwO/Bc4tN1ZHqLyktRvZnXP5E0t4LDpKAq6LDrCidvew3xt4D8WOEle6+9VlBxsuy20CxY6PFhxFqm4MczmKe5jIrtFRKmxHxyv1AHwtHw9aznJ7ANg8Oock4t3cyK7sjOkAlxXMBtau0gJVaO/xoDlm9soK19NmNq1xenaV/SY6gCTkt+zBBTzG0ursnFAR06tWXNDePa+zgM8CGwCTgX+lWMD6U+D80pJ1xqXRASQxT/E2Tmc8f+H/R0epkOujA/SlnXVed7v7Niu8NsPdt+3r56rEchtFsXnahOAokqL9uJ4d2A1r+SRKL9jZ8cq9AdbOyGu+mR1kZiMa10HAwsbPVfqGmWe+BLgiOock6tdM4cc8wLKePpHqL8Cd0SH60k55fZjiGLTngD83fvwRM1sVOLbEbJ1yWXQASdgf2ZozGMOcnn3k7NIq3u+C1g9mY2YjgU+6+9/38yUprFL+DbAUGBkdRBI1n7X5BmvyAa5nK/bAarkDcX9+Hh2gPy3/T3D3pcDuXcpSCs/8JeD30Tkkcc4IfsEULubuHtpqei5wVbtfbGbnm9lzZvaHEjP9VTv/gtxlZpeY2aFmduDyq/RknaWpo3TGg2zHWSxjPndHR+mCyx1fOPCX/dUFwD4lZVlJO+U1FngR2Av4+8b1/jJDleCS6ABSI3NYj9PZioe5ruZbTV80mC929xsonmrpilqvsG9mud0K7BSdQ2pmG27jAN6KsWZ0lA57GVjP8cWD+UVmthFwmbtvVUaoZgOuXzGzscCRwJY0ncjj7keUmKsM56Pykk67m514ipl8nFmswpbRcTro54Mtrm5rZ9r4X8AbgfdSrLSdDEnuRnkRsCA6hNRQsdX0pjxRzZXoQ/Tj6AADaXUAx/JR2Vvd/URgnrtfCOwHvKMb4TrJM58NTIvOITW1lNFcyBQu52acV6LjDNODVHALnBW1Gnnd2vj4auPjy2a2FbAGsG6pqcrzvegAUnO3sQvf5iUWJ3383recwd8MN7OLgJspThybaWZHdj5a05/X3w17M7vT3bc3s6MoFqptTfFW6OrAie7+3TKDlcVyuw/YIjqH1NwoFnI4tzGJPaKjDNJsYLLjc6ODDKTVyGtdMzsBGA8cTnEIx1TgFCCVraD7ck50AOkBSxjLuezBNcltNf2DFIoLWpfXSIpR1rima/WmK1U/pFg5LFK+6ezO95jFEv4YHaUNy4BvRYdo14DTxi7n6QrL7VvAp6JzSA8ZzTyOZgYTqfKRgZc5/T7HXDmtRl513vv9FKDSa1ikZl7lDZzDbkznBpxF0XH68c3oAIPRauS1lrt3bal/t1lu5wCfiM4hPWgyD/IxVmMkG0ZHafKA40m9kdXvyKvOxdXwH1DZfwGlzmayOaexBn/hlugoTb4aHWCweubZxr5YbmeTxoaKUlfv43p2ZFeM0YEpZgDbD2VtV6Re2lStLyfz2pbWIt13OVP4EQ+yjFmBKb6YWnFBj5eXZz4LrbqXaI+xNWcwNmir6Rsdvzzgzx22ni6vhq+jB7Yl2nzW4kx24B6uw+nmnvFf6OKf1VE9X16e+TPAd6JziADGNN7Ff3MPy3iuC3/eZY4nu0V6z5dXw3/QxR0gRVp6iO04C5jPjBL/lGXAv5X4+5dO5QV45i8An4vOIfJXc1iX09maB7m+pK2mf+z4vSX8vl3T00slmlluBtxA4qclSQ1tze18gDdjrNWh3/El4G2Od2NqWhqVVxPLbUvgLghdcyOysgnM4uO8xFg6sTf8EY7/oAO/TyhNG5t45vcBZ0TnEFnJy0ziNDbj8WFvNX1NHYoLNPJaieW2KnAfsHF0FpE+7cgtvI+3YawxyF+5ANja8RS25xmQRl4r8MwXoO1ypMpu551M5WUW8+Agf+VX6lJcoJFXvyy3i4EPRecQ6dcoFnIYtzG5ra2m7wJ2dnxJ2bG6RSOv/h0HXVkoKDI0SxjL99mDq/k9zrwWX7kUOKpOxQUqr3555s8CH6FYzCdSXTexG9/j2RZbTX/N8Tu7mqkLNG0cgOV2EvDF6BwiAxrNPI5iBuu+bqvpG4C9HO/m85JdofIagOU2ErgGmBKdRaQte3Eju7MTxjxgG8f/FB2pDCqvNlhu61Ns2JbqYbvSaybzAIfwGV/VfxMdpSy659WGxs4Tuv8l6ZjJL+pcXKDyaptnfjXFzqsiVXcV8OXoEGVTeQ1OBlwXHUKkhSeBQzzz2s8SVF6D4JkvBQ6CJE4/lt4zFzjQM38xOkg3qLwGyTN/HtgX6Im/IJKMVymKq3brufqj8hoCz/wR4AB08pBUgwOHNe7L9gyV1xB55r8HPoregZR4n/bML4oO0W0qr2HwzP8H7UAhsU72zL8ZHSKCymuYPPPvAF+KziE96fueedKHaAyHyqsDPPOvAd+IziE95RLg49EhIqm8OuczqMCkO64F/rmxdKdn6dnGDrPcTgT+PTqH1NavgH/yzBdFB4mmkVeHeeZfpdjIUP8qSKddAHxQxVVQeZXAMz8bOAyo1c6VEup04Ihenyo207SxRJbbAcB/A6tEZ5Gkfd4zPyU6RNWovEpmue1FcZ9i9egskpylwDGe+XnRQapI5dUFltvOFG9trxedRZKxCPiwZ/7z6CBVpXteXeCZ3wpsD9wUnUWSMBN4l4qrNZVXl3jms4B3AWcHR5FquxrYzjO/JTpI1WnaGMByOwQ4F1gtOotUhgMnAV/phY0EO0HlFcRy2xr4BfDW6CwS7iXgUM/88uggKdG0MYhnfi+wI8WNfOlddwA7qLgGT+UVyDOfDfwD8Dm0sWEv+i6wm2f+RHSQFGnaWBGW22bAecDu0VmkdH8CPu6Z/zo6SMo08qoIz/whYE/gWIqDFKSezge2VHENn0ZeFWS5bUgxpdgnOot0zOPAJzzzK6OD1IXKq8Ist49S7BG2VnQWGbJXKR6q/qpnviA6TJ2ovCrOclsXOAs4ODiKDN6NFM8m3h8dpI5UXomw3N5B8S+4buhX3/3Al/V4T7lUXolpbLNzCrBZdBZZyePAV4AfaZV8+VReCbLcRlKcGXkisHFwHIFZFI/2nOeZvxodpleovBJmuY0GDqc4eu1NwXF60YvA14GpuhnffSqvGrDcxlDc0D+W4pEjKdfTwHeAsz3zOdFhepXKq2Yst3dSlNiHgDHBcerEgWuAqcCl2ks+nsqrphpLLI4GjgEmB8dJ2csUp/Z82zN/ODaKNFN51ZzlNgo4APgUxWaIFhooHXcC5wAXeebzo8PIylRePcRymwR8ADgQmAKMjE1UOfcA04Bpnvnd0WGkNZVXj7Lc1qYYkR0I7E1v3h9bRnGuwDTgl575Y8F5ZBBUXoLlNh7Yj6LI9qLez1IuorjxPg24xDN/LjiPDJHKS17HcjNgc2DXxrUbsCnp3iubDdwMTG9ct2pNVj2ovGRAjSnmLrxWaDtQzUN0FwF3A7cDtzU+3q9HdepJ5SVDYrltQPF85aaNj28G/qZxTSjxj36e4hnCx4HHmn78OPCUHs/pHSov6bjGPbQNgTUpRmjLr3ErfL46xRsF8xvXvBU+Lv/xPIrnB5/wzLXLrAAqLxFJlPawF5EkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJKi8RSZLKS0SSpPISkSSpvEQkSSovEUmSyktEkqTyEpEkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJKi8RSZLKS0SSpPISkSSpvEQkSSovEUmSyktEkqTyEpEkqbxEJEkqLxFJkspLRJKk8hKRJKm8RCRJKi8RSZLKS0SSpPISkST9Lzn7oqp38seaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "### TARGET DISTRIBUTION OF SIMULATED DATA ###\n",
    "\n",
    "pd.value_counts(y).rename('Target Distribution').plot.pie(\n",
    "    colors=['green','lime'], figsize=(5,5)\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b313fc8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:29.821547Z",
     "iopub.status.busy": "2023-02-19T19:32:29.821174Z",
     "iopub.status.idle": "2023-02-19T19:32:29.826817Z",
     "shell.execute_reply": "2023-02-19T19:32:29.825961Z"
    },
    "papermill": {
     "duration": 0.01321,
     "end_time": "2023-02-19T19:32:29.828715",
     "exception": false,
     "start_time": "2023-02-19T19:32:29.815505",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### MODELING VARIABLE ###\n",
    "\n",
    "CV = StratifiedKFold(5, shuffle=True, random_state=1234)\n",
    "base_clf = XGBRFClassifier(random_state=1234)\n",
    "scoring = {\n",
    "    'fbeta': make_scorer(fbeta_score, beta=0.1), \n",
    "    'roc_auc':'roc_auc', \n",
    "    'average_precision':'average_precision'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "97c2652a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:32:29.840237Z",
     "iopub.status.busy": "2023-02-19T19:32:29.839595Z",
     "iopub.status.idle": "2023-02-19T19:34:05.035221Z",
     "shell.execute_reply": "2023-02-19T19:34:05.034026Z"
    },
    "papermill": {
     "duration": 95.204142,
     "end_time": "2023-02-19T19:34:05.037683",
     "exception": false,
     "start_time": "2023-02-19T19:32:29.833541",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### SIMPLE MODEL SELECTION ###\n",
    "\n",
    "base_model = RandomizedSearchCV(\n",
    "    base_clf, \n",
    "    dict(n_estimators=stats.randint(50,300)), \n",
    "    n_iter=20, random_state=1234,\n",
    "    cv=CV, n_jobs=-1, \n",
    "    refit=False, error_score='raise',\n",
    "    scoring=scoring,\n",
    ").fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cfa75420",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:34:05.047228Z",
     "iopub.status.busy": "2023-02-19T19:34:05.046886Z",
     "iopub.status.idle": "2023-02-19T19:36:48.090581Z",
     "shell.execute_reply": "2023-02-19T19:36:48.089511Z"
    },
    "papermill": {
     "duration": 163.051275,
     "end_time": "2023-02-19T19:36:48.092907",
     "exception": false,
     "start_time": "2023-02-19T19:34:05.041632",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### MODEL SELECTION WITH ThresholdClassifier ###\n",
    "\n",
    "threshold_model = RandomizedSearchCV(\n",
    "    ThresholdClassifier(base_clf), \n",
    "    dict(estimator__n_estimators=stats.randint(50,300)), \n",
    "    n_iter=20, random_state=1234,\n",
    "    cv=CV, n_jobs=-1, \n",
    "    refit=False, error_score='raise',\n",
    "    scoring=scoring,\n",
    ").fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d0e2a626",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:36:48.103126Z",
     "iopub.status.busy": "2023-02-19T19:36:48.101784Z",
     "iopub.status.idle": "2023-02-19T19:36:48.113224Z",
     "shell.execute_reply": "2023-02-19T19:36:48.112523Z"
    },
    "papermill": {
     "duration": 0.018838,
     "end_time": "2023-02-19T19:36:48.115681",
     "exception": false,
     "start_time": "2023-02-19T19:36:48.096843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### GET CV MEDIAN SCORE RESULTS ###\n",
    "\n",
    "base_model_test_roc_auc = np.median([\n",
    "    base_model.cv_results_[f'split{i}_test_roc_auc'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)\n",
    "base_model_test_avg_precision = np.median([\n",
    "    base_model.cv_results_[f'split{i}_test_average_precision'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)\n",
    "base_model_test_fbeta = np.median([\n",
    "    base_model.cv_results_[f'split{i}_test_fbeta'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)\n",
    "\n",
    "threshold_model_test_roc_auc = np.median([\n",
    "    threshold_model.cv_results_[f'split{i}_test_roc_auc'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)\n",
    "threshold_model_test_avg_precision = np.median([\n",
    "    threshold_model.cv_results_[f'split{i}_test_average_precision'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)\n",
    "threshold_model_test_fbeta = np.median([\n",
    "    threshold_model.cv_results_[f'split{i}_test_fbeta'] \n",
    "    for i in range(CV.n_splits)\n",
    "], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cb724597",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:36:48.124897Z",
     "iopub.status.busy": "2023-02-19T19:36:48.124549Z",
     "iopub.status.idle": "2023-02-19T19:36:48.412236Z",
     "shell.execute_reply": "2023-02-19T19:36:48.410947Z"
    },
    "papermill": {
     "duration": 0.29487,
     "end_time": "2023-02-19T19:36:48.414344",
     "exception": false,
     "start_time": "2023-02-19T19:36:48.119474",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7wAAAE9CAYAAAAoMG5jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwh0lEQVR4nO3dfZhcdZng/e+dBIxBXnxIHJGQdFhBE3m3A4zvwiiQnRG9hnXIBBFBMvLm6Dju4I66DM/memTGZ3YvENDG4UVswIgPmkcQcBxcBVHTISEQEI0QQsAdIwsIxAiBe/84p2Ol6U53p+t0dZ36fq6rrqr6nXN+dZ9Tp+quu86pX0VmIkmSJElS3UxqdQCSJEmSJFXBgleSJEmSVEsWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWLHglSZIkSbU0pdUBjIfp06dnV1dXq8OQJNXEihUrfpOZM1odRzszN0uSmmmo3NwRBW9XVxd9fX2tDkOSVBMR8XCrY2h35mZJUjMNlZs9pVmSJEmSVEsWvJIkSZKkWrLglSRJkiTVUkf8hleS6uj5559nw4YNbN68udWh1NbUqVOZOXMmO+20U6tDkSS1AXNz9Uabmy14JalNbdiwgV133ZWuri4iotXh1E5m8vjjj7NhwwbmzJnT6nAkSW3A3FytHcnNntIsSW1q8+bN7LnnnibUikQEe+65p9/SS5JGzNxcrR3JzRa8ktTGTKjVcvtKkkbL3FGt0W5fC16pw/T2QlcXTJpUXPf2Dt4mjcS6des44IADKun7+9//PhHBl7/85a1tq1atIiL4/Oc/P+J+RhJjlesh7Yixvi/7vi51LnPztioteCPi2Ih4ICLWRsS5g0yfFRG3RcTKiFgdEQsGmf5MRPztSPuUNLTeXli8GB5+GDKL6w99CE49ddu2xYv9cKSJ4YADDmDp0qVb71977bUcfPDBLYyo/ZmbJ77B3qtH87481uUlaXvaLTdXVvBGxGTgYuA4YB6wMCLmDZjt08DSzDwUOBG4ZMD0fwa+M8o+JQ3h7/8eNm3atu355+G557Zt27SpmFf1UtURny1btrBo0SLmzp3LCSecwKZNmzj//POZP38+BxxwAIsXLyYzAbjwwguZN28eBx10ECeeeCIAzz77LKeeeiqHH344hx56KN/61re29j179mw2b97Mv//7v5OZ3HzzzRx33HFbp69atYojjzySgw46iPe973088cQTAKxYsYKDDz6Ygw8+mIsvvnjr/C+88AKf/OQnmT9/PgcddBBf+tKXmrMR2oS5uT0M9l49mvflsS4vafyYm6vPzVUe4T0cWJuZD2bmc8B1wPED5klgt/L27sBj/RMi4r3AQ8CaUfYpaQjr11czrya+Ko/4PPDAA5x55pncf//97LbbblxyySWcffbZLF++nHvvvZff/e53fPvb3wbgc5/7HCtXrmT16tV88YtfBGDJkiUcddRR/PSnP+W2227jk5/8JM8+++zW/k844QS+/vWv86Mf/YjDDjuMl73sZVunnXzyyVxwwQWsXr2aAw88kH/4h38A4EMf+hAXXXQRd9999zax/su//Au77747y5cvZ/ny5Vx22WU89NBDY98I7cPc3AaGev8d6fvyWJeXND7MzYWqc3OVBe/ewCMN9zeUbY3OA06KiA3ATcA5ABHxCuDvgH/YgT4lDWHWrGrm1cRX5RGfffbZhze/+c0AnHTSSdx+++3cdtttHHHEERx44IH827/9G2vWFPXRQQcdxKJFi/jqV7/KlCnFP+PdeuutfO5zn+OQQw7hHe94B5s3b2Z9wyfz97///Xz961/n2muvZeHChVvbn3rqKZ588kne/va3A/DBD36QH/zgBzz55JM8+eSTvO1tbwPgAx/4wNZlbr31Vr7yla9wyCGHcMQRR/D444/zi1/8YuwboX2Ym9vAUO+/I31fHuvyksaHuZmtj1Vlbm71oFULgSszcyawALg6IiZRJNv/npnP7GjHEbE4Ivoiom/jxo3NiVZqc0uWwLRp27bttBPsvPO2bdOmFfOqPqo84jNwtMSI4Mwzz+T666/nnnvu4fTTT9/69wE33ngjZ511FnfddRfz589ny5YtZCbf+MY3WLVqFatWrWL9+vXMnTt3a3+vfvWr2Wmnnfjud7/L0UcfPaZYM5OLLrpo62M99NBDvPvd7x5TnzVkbm6xwd6rR/O+PNblJY0Pc3Oh6txcZcH7KLBPw/2ZZVuj04ClAJl5JzAVmA4cAfxjRKwDPgb8l4g4e4R9UvbXk5ndmdk9Y8aMMa+MVAeLFkFPD8yeDRHF9RVXwOWXb9vW01PMq/qo8ojP+vXrufPOOwG45ppreMtb3gLA9OnTeeaZZ7j++usBePHFF3nkkUd45zvfyQUXXMBTTz3FM888wzHHHMNFF1209bdEK1eufMljnH/++VxwwQVMnjx5a9vuu+/OK1/5Sn74wx8CcPXVV/P2t7+dPfbYgz322IPbb78dgN6Gc8OOOeYYLr30Up5//nkAfv7zn29zilYHMDe3gcHeq0fzvjzW5SWND3NzoercPKVpPb3UcmC/iJhDkfhOBP5ywDzrgaOBKyNiLkVS3ZiZb+2fISLOA57JzC9ExJQR9ClpOxYtGvxDjx+E6m3JkuJ3QY2nTjXriM/rXvc6Lr74Yk499VTmzZvHGWecwRNPPMEBBxzAq1/9aubPnw8Ug1KcdNJJPPXUU2QmH/3oR9ljjz34zGc+w8c+9jEOOuggXnzxRebMmbP1d0X93vSmNw362FdddRUf+chH2LRpE/vuuy9XXHEFAFdccQWnnnoqEbHNt8Qf/vCHWbduHYcddhiZyYwZM/jmN7859o3QPszNbWKo9+rxWl5S9czNhapzc/RX7VUo/8rgfwCTgcszc0lEnA/0ZeaychTHy4BXUAyS8Z8z89YBfZxHkVQ/P1Sfw8XR3d2dfX19TVsvSZoI7r///m1OLxpOb2/xu6D164tvj5cs8QPxSAy2nSNiRWZ2tyikMTE3S1J1zM3jYzS5udKCd6IwqUqqo9EmVe2YuhW8E4W5WVIdmZvHx2hyc6sHrZIkSZIkqRIWvJIkSZKkWrLglaQ21gk/S2klt68kabTMHdUa7fa14JWkNjV16lQef/xxE2tFMpPHH3+cqVOntjoUSVKbMDdXa0dyc5V/SyRJqtDMmTPZsGEDGzdubHUotTV16lRmzpzZ6jAkSW3C3Fy90eZmC15JalM77bQTc+bMaXUYkiSpZG6eeDylWZIkSZJUSxa8kiRJkqRasuCVJEmSJNWSBa8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEmSVEsWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWLHglSZIkSbVkwStJkiRJqiULXkmSJElSLVnwSpIkSZJqyYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1ZMErSZIkSaqlSgveiDg2Ih6IiLURce4g02dFxG0RsTIiVkfEgrL98IhYVV7ujoj3NSyzLiLuKaf1VRm/JEl1Y26WJHWSKVV1HBGTgYuBdwEbgOURsSwz72uY7dPA0sy8NCLmATcBXcC9QHdmbomIvYC7I+L/z8wt5XLvzMzfVBW7JEl1ZG6WJHWaKo/wHg6szcwHM/M54Drg+AHzJLBbeXt34DGAzNzUkECnlvNJkqSxMTdLkjpKlQXv3sAjDfc3lG2NzgNOiogNFN8gn9M/ISKOiIg1wD3ARxqSbAK3RsSKiFhcVfCSJNWQuVmS1FFaPWjVQuDKzJwJLACujohJAJn5k8x8AzAf+FRETC2XeUtmHgYcB5wVEW8brOOIWBwRfRHRt3HjxurXRJKkejA3S5Jqo8qC91Fgn4b7M8u2RqcBSwEy806KU6SmN86QmfcDzwAHlPcfLa9/DdxAcXrWS2RmT2Z2Z2b3jBkzxrwykiTVgLlZktRRqix4lwP7RcSciNgZOBFYNmCe9cDRABExlyKpbiyXmVK2zwZeD6yLiF0iYteyfRfg3RSDaEiSpOGZmyVJHaWyUZrLURzPBm4BJgOXZ+aaiDgf6MvMZcAngMsi4uMUv/85JTMzIt4CnBsRzwMvAmdm5m8iYl/ghojoj/2azLy5qnWQJKlOzM2SpE4TmfUfZLG7uzv7+vxbQElSc0TEiszsbnUc7czcLElqpqFyc6sHrZIkSZIkqRIWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWLHglSZIkSbVkwStJkiRJqiULXkmSJElSLVnwSpIkSZJqyYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFb4fp7YWuLpg0qbju7d1+uyRJksaPn8lGx+2l4UxpdQAaP729sHgxbNpU3H/44eL+HXfAVVe9tB1g0aLWxCpJktRphvqsBn4mG4zbSyMRmdnqGCrX3d2dfX19rQ6j5bq6ijeCgSZPhhdeeGn77Nmwbl3VUUlS+4mIFZnZ3eo42pm5WXqpoT6r+ZlscG4vNRoqN3tKcwdZv37w9sGK3e3NL0mSpOYb6rOXn8kG5/bSSFjwdpBZswZvnzx5dPNLkiSp+Yb67OVnssG5vTQSFrwdZMkSmDZt27Zp04rfOgzWvmTJ+MUmSZLU6Yb6rOZnssG5vTQSFrwdZNEi6OkpftcQUVz39MAllwze7o/9JUmSxs9Qn9X8TDY4t5dGwkGrJEkaJQetGjtzsySpmRy0SpIkSZLUUSx4JUmSJEm1ZMErSZIkSaolC15JkiRJUi1Z8EqSJEmSasmCV5IkSZJUSxa8kiRJkqRasuCVmqi3F7q6YNKk4rq3t7nzN1unP74kaWKrIk/09xkBU6YU1wP7Nj9J9VFpwRsRx0bEAxGxNiLOHWT6rIi4LSJWRsTqiFhQth8eEavKy90R8b6R9im1Sm8vLF4MDz8MmcX14sVDJ8nRzt/qeOv2+FKnMjerXVSRJxr7BHjhheK6sW/zk1QvkZnVdBwxGfg58C5gA7AcWJiZ9zXM0wOszMxLI2IecFNmdkXENOC5zNwSEXsBdwOvAXK4PgfT3d2dfX19zV9JqUFX1x8SaKPZs2HdurHP32yd/vjSWETEiszsbnUco2VuVjupIk8M1Wdj32B+ktrRULm5yiO8hwNrM/PBzHwOuA44fsA8CexW3t4deAwgMzdl5payfWo530j7lFpi/fpq25ut0x9f6lDmZrWNKvLEcMuuX29+kuqmyoJ3b+CRhvsbyrZG5wEnRcQG4CbgnP4JEXFERKwB7gE+UibZkfQptcSsWdW2N1unP77UoczNahtV5Inhlp01y/wk1U2rB61aCFyZmTOBBcDVETEJIDN/kplvAOYDn4qIqaPpOCIWR0RfRPRt3Lix6YFLAy1ZAtOmbds2bVrR3oz5m63TH1/SkMzNmhCqyBOD9Tmwb/OTVC9VFryPAvs03J9ZtjU6DVgKkJl3UpwiNb1xhsy8H3gGOGCEffYv15OZ3ZnZPWPGjDGshjQyixZBT0/xG5+I4rqnp2hvxvytjrdujy91KHOz2kYVeaKxT4DJk4vrxr7NT1K9VDlo1RSKQSyOpkh8y4G/zMw1DfN8B/haZl4ZEXOB71GcBtUFPFIOjDEbuBM4CHhyuD4H48AYkqRmauNBq8zNkqRaGio3T6nqAcuEeDZwCzAZuDwz10TE+UBfZi4DPgFcFhEfpxj84pTMzIh4C3BuRDwPvAicmZm/KVfkJX1WtQ6SJNWJuVmS1GkqO8I7kfgtsiSpmdr1CO9EYm6WJDVTK/6WSJIkSZKklrHglSRJkiTVkgWvJEmSJKmWLHglSZIkSbVkwStJkiRJqiULXkmSJElSLVnwSpIkSZJqyYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1ZMErSZIkSaolC15JkiRJUi1Z8EqSJEmSasmCV5IkSZJUSxa8kiRJkqRasuCVJEmSJNWSBa8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEmSVEsWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWLHglSZIkSbVUacEbEcdGxAMRsTYizh1k+qyIuC0iVkbE6ohYULa/KyJWRMQ95fVRDct8v+xzVXl5VZXrIElSnZibJUmdZEpVHUfEZOBi4F3ABmB5RCzLzPsaZvs0sDQzL42IecBNQBfwG+DPMvOxiDgAuAXYu2G5RZnZV1XskiTVkblZktRpqjzCeziwNjMfzMzngOuA4wfMk8Bu5e3dgccAMnNlZj5Wtq8BXh4RL6swVkmSOoG5WZLUUaosePcGHmm4v4FtvwkGOA84KSI2UHyDfM4g/fw5cFdm/r6h7YrylKnPREQ0MWZJkurM3CxJ6iitHrRqIXBlZs4EFgBXR8TWmCLiDcAFwF81LLMoMw8E3lpePjBYxxGxOCL6IqJv48aNla2AJEk1Y26WJNXGiAveiHhVOZDFrIiYNYJFHgX2abg/s2xrdBqwFCAz7wSmAtPLx5sJ3ACcnJm/7F8gMx8tr58GrqE4PeslMrMnM7szs3vGjBkjWUVJktqKuVmSpO0btuCNiPdExC+Ah4D/CawDvjOCvpcD+0XEnIjYGTgRWDZgnvXA0eXjzKVIqhsjYg/gRuDczLyjIZYpEdGfdHcC/hS4dwSxSJJUG+ZmSZJGZiRHeP9v4Ejg55k5hyIJ/ni4hTJzC3A2xSiO91OM+LgmIs6PiPeUs30COD0i7gauBU7JzCyXey3w2QF/cfAy4JaIWA2sovhW+rKRr64kSbVgbpYkaQSiyGHbmSGiLzO7y8R3aGa+GBF3Z+bB4xPi2HV3d2dfn/+UIElqjohYkZndLXx8c7MkSQ2Gys0j+R/eJyPiFcAPgN6I+DXwbLMDlCRJI2ZuliRpBEZySvPxwCbg48DNwC8pfp8jSZJaw9wsSdIIjKTg/WxmvpiZWzLzqsy8EPi7qgOTJElDMjdLkjQCIyl43zVI23HNDkSSJI2YuVmSpBEY8je8EXEGcCawbznyYr9dgTsGX0qSJFXF3CxJ0uhsb9Cqayj+0+//Ac5taH86M/93pVFJkqTBmJslSRqFIU9pzsynMnNdZi4E9gGOysyHgUkRMWfcIpRUG7290NUFkyYV1729rY5o9OqwDp2mTs+ZuVlqjVa8j9TpvUsaaDz372H/ligi/ivQDbwOuALYGfgq8ObqwpJUN729sHgxbNpU3H/44eI+wKJFrYtrNOqwDp2mrs+ZuVkaP614H6nre5cE479/R2Zuf4aIVcChwF2ZeWjZtjozD2p+ONXwz+2l1uvqKt7QBpo9G9atG+9odkwd1qHTVPWcDfXn9uPF3CyNn1a895tvVGfjnZtHMkrzc1lUxVl2tMuOhyGpU61fP7r2iagO69BpavycmZulcdKK95Eav3dJ475/j6TgXRoRXwL2iIjTgX8FLqsmHEl1NWvW6NonojqsQ6ep8XNmbpbGSSveR2r83iWN+/49bMGbmZ8Hrge+AexP8Wf3F1UTjqS6WrIEpk3btm3atKK9XdRhHTpNXZ8zc7M0flrxPlLX9y4Jxn//HskRXoB7gB8CPyhvS9KoLFoEPT3F7zMiiuuenvYafKMO69Bpav6cmZulcdCK95Gav3epw433/j2SQas+DHwW+DcggLcD52fm5dWE1HwOjCFJaqYJMGiVuVmSpAZD5eZh/5YI+CRwaGY+Xna0J/AjoG2SqiRJNWNuliRpBEZySvPjwNMN958u2yRJUmuYmyVJGoEhj/BGxN+UN9cCP4mIb1H8/cHxwOpxiE2SJDUwN0uSNDrbO8J7KLAr8GfANyn/6w/4FvBQtWFJkqRBmJslSRqF7f2G9zDg74A/B/yrA0mSWs/cLEnSKGyv4P0S8D1gDtA4jGJQfKO8b4VxSZKklzI3S5I0CkOe0pyZF2bmXOCKzNy34TInM02okiSNM3OzJEmjM+wozZl5xngEIkmSRsbcLEnSyIzkb4kkSZIkSWo7FrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1VGnBGxHHRsQDEbE2Is4dZPqsiLgtIlZGxOqIWFC2vysiVkTEPeX1UQ3LvLFsXxsRF0ZEVLkOkiTViblZktRJKit4I2IycDFwHDAPWBgR8wbM9mlgaWYeCpwIXFK2/wb4s8w8EPggcHXDMpcCpwP7lZdjq1oHSZLqxNwsSeo0VR7hPRxYm5kPZuZzwHXA8QPmSWC38vbuwGMAmbkyMx8r29cAL4+Il0XEXsBumfnjzEzgK8B7K1wHSZLqxNwsSeooUyrse2/gkYb7G4AjBsxzHnBrRJwD7AL8ySD9/DlwV2b+PiL2Lvtp7HPvpkUsSVK9mZslSR2l1YNWLQSuzMyZwALg6ojYGlNEvAG4APir0XYcEYsjoi8i+jZu3Ni0gCVJqjlzsySpNqoseB8F9mm4P7Nsa3QasBQgM+8EpgLTASJiJnADcHJm/rKhz5nD9EnZX09mdmdm94wZM8a4KpIk1YK5WZLUUaoseJcD+0XEnIjYmWLgi2UD5lkPHA0QEXMpkurGiNgDuBE4NzPv6J85M38F/DYijixHgDwZ+FaF6yBJUp2YmyVJHaWygjcztwBnA7cA91OM+LgmIs6PiPeUs30COD0i7gauBU4pB7w4G3gt8NmIWFVeXlUucybwZWAt8EvgO1WtgyRJdWJuliR1mkp/w5uZN2Xm/pn5HzJzSdn22cxcVt6+LzPfnJkHZ+YhmXlr2f7fMnOXsq3/8utyWl9mHlD2eXaZhFuitxe6umDSpOK6t7dVkUgTz3i9PvofJwKmTCmufT2Oj4HP8Zln+p7YDuqSm83Bqpr7mCaK4fbFHcnHHbV/Z2btL2984xuz2b761cxp0zLhD5dp04p2qdON1+tjsMfx9Tg+trftO+E5APpyAuS3dr6MJTebg1U19zFNFMPtizuSj+u6fw+Vm6OYVm/d3d3Z19fX1D67uuDhh1/aPns2rFvX1IeS2s54vT6GepyqHk9/MNy271fX5yAiVmRmd6vjaGdjyc3mYFXNfUwTxXD74o7k47ru30PlZgveHTRpUvF9yEAR8OKLTX0oqe2M1+tjqMep6vH0B8Nt+351fQ4seMduLLnZHKyquY9pohhuX9yRfFzX/Xuo3Nzq/+FtW7Nmja5d6iTj9foYrj9fj9UZ6bb1OVAVzMGqmvuYJorh9sUdycedtn9b8O6gJUtg2rRt26ZNK9qlTjder4/BHqfKx9MfbG/b9/M5UFXMwaqa+5gmiuH2xR3Jx522f1vw7qBFi6CnpzjXPaK47ukp2qVON16vj8bHAZg8ubj29Vi9wZ7jM87wPVHjwxysqrmPaaIYbl/ckXzcafu3v+GVJGmU/A3v2JmbJUnN5G94JUmSJEkdxYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1ZMErSZIkSaolC15JkiRJUi1Z8EqSJEmSasmCV5IkSZJUSxa8kiRJkqRasuCVJEmSJNWSBa8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEmSVEsWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWKi14I+LYiHggItZGxLmDTJ8VEbdFxMqIWB0RC8r2Pcv2ZyLiCwOW+X7Z56ry8qoq10GSpDoxN0uSOsmUqjqOiMnAxcC7gA3A8ohYlpn3Ncz2aWBpZl4aEfOAm4AuYDPwGeCA8jLQoszsqyp2SZLqyNwsSeo0VR7hPRxYm5kPZuZzwHXA8QPmSWC38vbuwGMAmflsZt5OkVwlSVJzmJslSR2lyoJ3b+CRhvsbyrZG5wEnRcQGim+Qzxlh31eUp0x9JiJizJFKktQZzM2SpI7S6kGrFgJXZuZMYAFwdUQMF9OizDwQeGt5+cBgM0XE4ojoi4i+jRs3NjVoSZJqzNwsSaqNKgveR4F9Gu7PLNsanQYsBcjMO4GpwPTtdZqZj5bXTwPXUJyeNdh8PZnZnZndM2bM2KEVkCSpZszNkqSOUmXBuxzYLyLmRMTOwInAsgHzrAeOBoiIuRRJdcivfCNiSkRML2/vBPwpcG8FsUuSVEfmZklSR6lslObM3BIRZwO3AJOByzNzTUScD/Rl5jLgE8BlEfFxikEyTsnMBIiIdRSDZuwcEe8F3g08DNxSJtTJwL8Cl1W1DpIk1Ym5WZLUaaLMYbXW3d2dfX3+U4IkqTkiYkVmdrc6jnZmbpYkNdNQubnVg1ZJkiRJklQJC15JkiRJUi1Z8EqSJEmSasmCV5IkSZJUSxa8kiRJkqRasuCVJEmSJNWSBa8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEmSVEsWvJIkSZKkWrLglSRJkiTVkgWvJEmSJKmWLHglSZIkSbVkwStJkiRJqiULXkmSJElSLVnwSpIkSZJqyYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1ZMErSZIkSaolC15JkiRJUi1Z8EqSJEmSaqnSgjcijo2IByJibUScO8j0WRFxW0SsjIjVEbGgbN+zbH8mIr4wYJk3RsQ9ZZ8XRkRUuQ6SJNWJuVmS1EkqK3gjYjJwMXAcMA9YGBHzBsz2aWBpZh4KnAhcUrZvBj4D/O0gXV8KnA7sV16ObX70kiTVj7lZktRpqjzCeziwNjMfzMzngOuA4wfMk8Bu5e3dgccAMvPZzLydIrluFRF7Abtl5o8zM4GvAO+tbhUkSaoVc7MkqaNMqbDvvYFHGu5vAI4YMM95wK0RcQ6wC/AnI+hzw4A+9x5bmJIkdQxzsySpo7R60KqFwJWZORNYAFwdEU2JKSIWR0RfRPRt3LixGV1KktQJzM2SpNqosuB9FNin4f7Msq3RacBSgMy8E5gKTB+mz5nD9EnZX09mdmdm94wZM0YZuiRJtWRuliR1lCoL3uXAfhExJyJ2phj4YtmAedYDRwNExFyKpDrkV76Z+SvgtxFxZDkC5MnAt6oIXpKkGjI3S5I6SmW/4c3MLRFxNnALMBm4PDPXRMT5QF9mLgM+AVwWER+nGCTjlHLACyJiHcWgGTtHxHuBd2fmfcCZwJXAy4HvlBdJkjQMc7MkqdNEmcNqrbu7O/v6+lodhiSpJiJiRWZ2tzqOdmZuliQ101C5udWDVkmSJEmSVAkLXkmSJElSLVnwSpIkSZJqyYJXkiRJklRLFrySJEmSpFqy4JUkSZIk1ZIFryRJkiSplix4JUmSJEm1ZMErSZIkSaolC15JkiRJUi1Z8EqSJEmSasmCV5IkSZJUSxa8kiRJkqRasuCVJEmSJNWSBa8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfCOQG8vdHXBpEnFdW9vqyPSjqrrc1nX9ZIkaaJr1xw80rjbdf2kflNaHcBE19sLixfDpk3F/YcfLu4DLFrUurg0enV9Luu6XpIkTXTtmoNHGne7rp/UKDKz1TFUrru7O/v6+nZo2a6u4sU90OzZsG7dmMLSOKvrc1nX9ZImsohYkZndrY6jnY0lN0sTRbvm4JHG3a7rp840VG72lOZhrF8/unZNXHV9Luu6XpIkTXTtmoNHGne7rp/UyIJ3GLNmja5dE1ddn8u6rpckSRNdu+bgkcbdrusnNbLgHcaSJTBt2rZt06YV7WovdX0u67pekiRNdO2ag0cad7uun9TIgncYixZBT0/xW4WI4rqnxx/qt6O6Ppd1XS9Jkia6ds3BI427XddPauSgVZIkjZKDVo2duVmS1EwOWiVJkiRJ6igWvJIkSZKkWqq04I2IYyPigYhYGxHnDjJ9VkTcFhErI2J1RCxomPapcrkHIuKYhvZ1EXFPRKyKCM+FkiRpFMzNkqROMqWqjiNiMnAx8C5gA7A8IpZl5n0Ns30aWJqZl0bEPOAmoKu8fSLwBuA1wL9GxP6Z+UK53Dsz8zdVxS5JUh2ZmyVJnabKI7yHA2sz88HMfA64Djh+wDwJ7Fbe3h14rLx9PHBdZv4+Mx8C1pb9SZKkHWduliR1lCoL3r2BRxrubyjbGp0HnBQRGyi+QT5nBMsmcGtErIiIxc0OWpKkGjM3S5I6SqsHrVoIXJmZM4EFwNURMVxMb8nMw4DjgLMi4m2DzRQRiyOiLyL6Nm7c2NyoJUmqL3OzJKk2qix4HwX2abg/s2xrdBqwFCAz7wSmAtO3t2xm9l//GriBIU6nysyezOzOzO4ZM2aMeWUkSaoBc7MkqaNEZlbTccQU4OfA0RQJcTnwl5m5pmGe7wBfy8wrI2Iu8D2K06PmAddQJMzXlO37USTdSZn5dETsAnwXOD8zbx4mlo3Aw01eRRWmAw5SUi23cfXcxtWq4/adnZltV7GZm4fVzvuqsbeGsbeGsbfGRI990Nxc2SjNmbklIs4GbgEmA5dn5pqIOB/oy8xlwCeAyyLi4xS//zkliwp8TUQsBe4DtgBnZeYLEfFHwA0R0R/7NcMl1DKWtvtQ0i4ioi8zu1sdR525javnNq6W23fiMDdvXzvvq8beGsbeGsbeGu0ae2VHeNUZ2nXHbydu4+q5javl9lW7aOd91dhbw9hbw9hbo11jb/WgVZIkSZIkVcKCV2PV0+oAOoDbuHpu42q5fdUu2nlfNfbWMPbWMPbWaMvYPaVZkiRJklRLHuGVJEmSJNWSBa+2iohjI+KBiFgbEecOMn1WRNwWESsjYnVELCjbF0XEqobLixFxSDntjRFxT9nnhVEO49mpKtrGSyLikYh4ZpxXZ0Jq9jaOiGkRcWNE/Cwi1kTE58Z/rSaWivbjmyPi7nIbfzEiJo/zaqmGhttXy3neHxH3lfveNQ3tN0fEkxHx7QHz/7BhH34sIr7ZRrEfHRF3lbHfHhGvbZO4jyrjvjcirori77WabkdjL3PFnWXb6oj4i4b550TET8o+vxYRO7dR7GeX/WVETK8i7gpj7y37vDciLo+Indoo9n8p8+HqiLg+Il7RLrE3LHdhTKTPpZnpxQsUf0/xS2BfYGfgbmDegHl6gDPK2/OAdYP0cyDwy4b7PwWOBAL4DnBcq9e1htv4SGAv4JlWr2OrL1VsY2Aa8M7y9s7AD92PK9mPdyuvA/gGcGKr19VLe19GuK/uB6wEXlnef1XDtKOBPwO+vZ3H+AZwcrvETvEfzHPL22cCV070uCkOzjwC7F/ePx84bSJtc2B/YL/y9muAXwF7lPeX9r+fAV/sf29sk9gPBbqAdcD0ZsddcewLKPJJANe22XbfrWH5fwbObZfYy7Zu4Gom0OdSj/Cq3+HA2sx8MDOfA64Djh8wTwK7lbd3Bx4bpJ+F5bJExF4UL9ofZ/EK+Arw3gpibxdN38YA5fb9VQXxtqOmb+PM3JSZt5W3nwPuAmZWEHu7qGo//m15cwpF8nWACY3VSPbV04GLM/MJgMz8df+EzPwe8PRQnUfEbsBRwDebHDdUF/tIXpsTLe49gecy8+fl/e8Cf97kuMcUe2b+PDN/Ud5+DPg1MCMigmIfub5c/iqq+RzU9NjL+yszc10F8Y5H7DdlieLgSxV5u6rYfwtQ7j8vp5p8WEnsUZyd9U/Af64g5h1mwat+e1N8g9pvQ9nW6DzgpIjYANwEnDNIP39B8U1af58bhumzk1SxjbWtSrdxROxBcfThe02ItV1Vto0j4haKxPk0f/iAKO2okeyr+wP7R8QdEfHjiDh2FP2/F/hew5c1zVRV7B8Gbipfmx8Amv0TjSri/g0wJSL6//vzBGCfpkS7rabEHhGHU3xp90uKYv3JzNyynT6boYrYx0ulsZenMn8AuLmpURcqiz0irgD+F/B64KJmB051sZ8NLJtoB2IseDUaCylOf5pJcarI1RGxdR+KiCOATZl5b6sCrAG3cfV2aBtH8Zuxa4ELM/PB8Qy4De3QNs7MYyhOz38ZxVERqWpTKE7bewfFfntZ+cXWSCyktV8+7kjsHwcWlK/NKyhOlxxvo4q7PEJ3IvDfI+KnFF+IvVB9mIPabuzlmW1XAx/KzBdbEeB2dGrslwA/yMwfjk+oL7FDsWfmhyhOF76f4gviVhhV7BHxGuA/UU2BPiYWvOr3KNt+YzqzbGt0GsVvUcjMO4GpQOMgBieybfJ/lG1PIRmsz05SxTbWtqrcxj3ALzLzfzQr2DZV6X6cmZuBb/HSU6uk0RrJvrqB4mjE85n5EMVvXPcbruNyAJ/DgRubFOtATY89ImYAB2fmT8qmrwFval7IQEXbPDPvzMy3ZubhwA/KZZptTLGXp7jfCPx9Zv64nP9xYI/4wyBbVX0OqiL28VJZ7BHxXylOtf2bdosdIDNfoDjVuIpT+KuI/VDgtcDaiFgHTIuItRXEPmoWvOq3HNgvitEEd6b4QLpswDzrKQaUICLmUnyI3VjenwS8n21/k/cr4LcRcWT5O4STKT7Idqqmb2O9RCXbOCL+G8Xv3T5WZfBtounbOCJeUX5T3H8k/T8CP6t4PVR/I9lXv0lx9KK/iN0fGMkZHCdQDKy0uWnRbquK2J8Ado+I/cv776I4etRMlWzziHhVef0y4O8oBn9qth2OvZz/BuArmbn15xjl0enbKPYXgA9Szeegpsc+jiqJPSI+DBwDLKzwiHXTY4/Ca/tvA++hmnxYxf5+Y2a+OjO7MrOL4kyupo8Ev0NyAoyc5WViXChOPfw5xXn4f1+2nQ+8p7w9D7iDYiS3VcC7G5Z9B/DjQfrsBu4t+/wCEK1ezxpu43+k+BbuxfL6vFavZ522McW3nknxwXBVeflwq9ezZtv4jyiS7+ry/eIiYEqr19NL+19GsK8GxWm99wH30DA6OMWI7BuB35Xvrcc0TPs+cGy7xQ68r5z37nId9m2TuP+pfA9+APjYRNvmwEnA8w05YhVwSDltX4pBk9YCXwde1kaxf7R8HrZQDHD25TaKfUvZX3/7Z9shdoqDkXeU894L9NIwavNEjn2Q/ifMKM1RBiRJkiRJUq14SrMkSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEnqIBHRHREXbmf6ayKiFf/LKzWdf0skSZIkjUFETM7MFzr18aWJzCO8UgeJiG9GxIqIWBMRi8u2ZxqmnxARV5a3/ygiboiIu8vLm1oUtiRJTTNELvxIRPxTwzynRMQXytsnRcRPI2JVRHwpIiaX7c9ExP8bEXcDfxwRn42I5RFxb0T0RESU882PiNXl8v8UEfeW7ZPL+8vL6X81SKxdEfGziOiNiPsj4vqImFZOWxcRF0TEXcB/ioh3R8SdEXFXRHw9Il7R8Pg/KnP5TyNi14h4R0R8u5z+9jK2VRGxspze1RDn1Ii4IiLuKae/s2Eb/X8RcXNE/CIi/rGq50waCwteqbOcmplvBLqBj0bEntuZ90Lgf2bmwcBhwJrxCFCSpIoNlgu/AbyvYZ6/AK6LiLnl7Tdn5iHAC8Cicp5dgJ9k5sGZeTvwhcycn5kHAC8H/rSc7wrgrxqW73ca8FRmzgfmA6dHxJxB4n0dcElmzgV+C5zZMO3xzDwM+Ffg08CflPf7gL+JiJ2BrwF/XebzPwF+N6D/vwXOKuN76yDTzwIyMw8EFgJXRcTUctoh5fY5EPiLiNhnkPillrLglTrLR8tvon8M7APst515jwIuBcjMFzLzqXGIT5Kkqr0kF2bmRuDBiDiyLIBfD9wBHA28EVgeEavK+/uW/bxAUSj3e2dE/CQi7qHIoW+IiD2AXTPzznKeaxrmfzdwctnvT4A9GTwvP5KZd5S3vwq8pWHa18rrI4F5wB1lfx8EZlMUy7/KzOUAmfnbzNwyoP87gH+OiI8Cewwy/S3l45KZPwMeBvYvp30vM5/KzM3AfeVjShPKlFYHIGl8RMQ7KL7Z/ePM3BQR3wemAo0/5J/60iUlSaqH7eRCgOuA9wM/A27IzCxPS74qMz81SHeb+383Wx7xvATozsxHIuI8hs+pAZyTmbcMM9/AAXca7z/b0Nd3M3PhNg8QceAwfZOZn4uIG4EFFAXzMcDm4ZYr/b7h9gtYW2gC8giv1Dl2B54oE/zrKb4NBvj3iJgbEZPY9nSu7wFnwNbfGe0+vuFKktR0Q+VCgBuA4ylO272ubPsecEJEvAogIv6viBjsKGZ/cfub8rezJwBk5pPA0xFxRDn9xIZlbgHOiIidyr73j4hdBul7VkT8cXn7L4HbB5nnx8CbI+K1ZV+7RMT+wAPAXhExv2zfNSK2KUoj4j9k5j2ZeQGwnOLodqMfUp7GXfY5q+xXagsWvFLnuBmYEhH3A5+jSI4A5wLfBn4E/Kph/r+mOD3rHmAFxalSkiS1s6FyIZn5BHA/MDszf1q23Ufx29hbI2I18F1gr4GdloXtZcC9FIXs8obJpwGXlaca7wL0/0ToyxSnAd9VDhD1JQY/QvoAcFYZ8yspf2404PE3AqcA15Zx3gm8PjOfo/iN7UXladzf5aVHnj9WDrS1Gnge+M6A6ZcAk8rPA18DTsnM3yO1Cf+WSJIkSapIRLwiM58pb58L7JWZfz3CZbuAb5cDYUnaAZ5nL0mSJFXnP0bEpyg+dz9McSRW0jjxCK8kSZIkqZb8Da8kSZIkqZYseCVJkiRJtWTBK0mSJEmqJQteSZIkSVItWfBKkiRJkmrJgleSJEmSVEv/B8BeZrigMEsEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PLOT SIMPLE MODEL SEARCH RESULTS ###\n",
    "\n",
    "plt.figure(figsize=(16,5))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.scatter(\n",
    "    base_model_test_roc_auc,\n",
    "    base_model_test_fbeta,\n",
    "    c='blue', label='baseModel'\n",
    ")\n",
    "plt.ylabel('fbeta'); plt.xlabel('auc'); plt.legend()\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(\n",
    "    base_model_test_avg_precision,\n",
    "    base_model_test_fbeta,\n",
    "    c='blue', label='baseModel'\n",
    ")\n",
    "plt.ylabel('fbeta'); plt.xlabel('average precision'); plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "69926646",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:36:48.423971Z",
     "iopub.status.busy": "2023-02-19T19:36:48.423604Z",
     "iopub.status.idle": "2023-02-19T19:36:48.706596Z",
     "shell.execute_reply": "2023-02-19T19:36:48.705137Z"
    },
    "papermill": {
     "duration": 0.291255,
     "end_time": "2023-02-19T19:36:48.709668",
     "exception": false,
     "start_time": "2023-02-19T19:36:48.418413",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7wAAAE9CAYAAAAoMG5jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtl0lEQVR4nO3de7RddXno/e+ThBCjQWyISA0kUEESAQNsc0CKoXgBPZZbUcgbFShtWtBjexxYYPC+vXDM8Ea1w1bPaaxc1K2gtCoDW6kHoVwOl2wkhJvEgAkEsMYcUDAgEp73jzl3WAl7J3sna+611299P2Ossdb6zct65m/NNZ/1rHlZkZlIkiRJklSaCZ0OQJIkSZKkJljwSpIkSZKKZMErSZIkSSqSBa8kSZIkqUgWvJIkSZKkIlnwSpIkSZKKNKnTAYyF3XbbLWfPnt3pMCRJhbjjjjt+npkzOh1HNzM3S5Laabjc3BMF7+zZsxkYGOh0GJKkQkTEmk7H0O3MzZKkdhouN3tIsyRJkiSpSBa8kiRJkqQiNVrwRsSxEfFARKyKiPOGGD4rIq6NiBURcX1EzGxp/2FELI+IeyPiT1umOTQi7q7n+bmIiCaXQZKkkpibJUm9pLFzeCNiIvB54O3AWmBZRFyVmfe1jHYR8OXMvCwijgY+DrwfeBw4PDN/HRGvAO6pp30M+J/AHwO3Af8KHAv8W1PLIUk74je/+Q1r167l2Wef7XQo2g5Tpkxh5syZ7LTTTp0OpS3MzZJkbu52o83NTV60aj6wKjMfAoiIy4HjgdakOhf4SP34OuDbAJn5XMs4O1PviY6IPYBdMvPW+vmXgRMwqUoap9auXcu0adOYPXs27vTqLpnJ+vXrWbt2LXvvvXenw2kXc7Oknmdu7l7bk5ubPKT5tcAjLc/X1m2t7gJOqh+fCEyLiOkAEbFnRKyo5/HJ+hfk19bz2do8qadfHBEDETGwbt26HV4YSdoezz77LNOnTzehdqGIYPr06aXtATA3S+p55ubutT25udMXrToHWBARdwILgEeBjQCZ+UhmHgS8DjgtInYfzYwzc2lm9mVm34wZ/lWipM4xoXavHn3vzM2Sitej2/cijPa9a7LgfRTYs+X5zLptk8x8LDNPysyDgQvqtie3HAe4Bziynn7m1uYpdZX+fpg9GyZMqO77+zvz+hEwaVJ1P5ZxdHr5e8CTTz7JF77wBQCuv/563v3ud7f9NU4//XSuvPLKEY+/evVqDjjggCGHHXXUUZv+m3X27NkceeSRmw2fN2/esNMOp3WeOzJOIczNvaSEbWxTy9DfD7vtVuW9iOpxN/aPupK5eWxzc5MF7zJg34jYOyImA6cCV7WOEBG7RcRgDOcDF9ftMyPiZfXjVwG/CzyQmY8Dv4yIw+orQH4A+E6DyyA1p78fFi+GNWsgs7pfvHhsi83B1wfYuLG6H6s4Or38PaI1qY7UxsF1YRx46qmneOSR6gjc+++/v8PRFMHc3CtK2MY2tQz9/XDGGbB+/Ytt69fDH/5hd/WPupa5eWw1VvBm5vPAh4BrgPuBb2TmvRFxYUQcV492FPBARKwEdgeW1O1zgNsi4i7gP4CLMvPuetjZwD8Bq4AH8aIY6lYXXAAbNmzetmFD1d6p1x/LODq9/ONVm/dmnHfeeTz44IPMmzePj370ozz99NOcfPLJ7L///ixatIjMBKpfbM8991wOOeQQvvnNb/Lv//7vHH744RxyyCG85z3v4emnn940v7lz53LQQQdxzjnnbHqdG264gTe/+c3ss88+m35Rzkw++tGPcsABB3DggQdyxRVXvCS+Z555hlNPPZU5c+Zw4okn8swzz2w2/L3vfe+m6b7+9a+zcOHCTcOeffZZzjjjDA488EAOPvhgrrvuum3Oc7jl6hXm5h5Swja2qWW44AL4zW9e2v7cc93VPxo75ubNhnddbs7M4m+HHnpoSuNORGb1m/Xmt4jOvv5YxdHp5R8j991338hH/upXM6dO3bw/pk6t2rfTT37yk3zDG96QmZnXXXdd7rLLLvnII4/kxo0b87DDDssbb7wxMzNnzZqVn/zkJzMzc926dXnkkUfm008/nZmZn/jEJ/Jv/uZv8uc//3nut99++cILL2Rm5hNPPJGZmaeddlqefPLJuXHjxrz33nvzd37ndzIz88orr8y3ve1t+fzzz+dPf/rT3HPPPfOxxx7bLKa//du/zTPOOCMzM++6666cOHFiLlu2bFNMP/rRj/Lwww/PzMx58+blvffeu2naiy66aNO0999/f+655575zDPPDDvP4ZYrM3PBggWbXndLQ72HwECOg/zWzTdzc8NK2MY2tQxby3/d1D/abubm3srNnb5oldS79tprdO1j9fpjFUenl388GoM9MvPnz2fmzJlMmDCBefPmsXr16k3DTjnlFABuvfVW7rvvPo444gjmzZvHZZddxpo1a3jlK1/JlClTOPPMM/mXf/kXpk6dumnaE044gQkTJjB37lz+8z//E4CbbrqJhQsXMnHiRHbffXcWLFjAsmXLNovnhhtu4H3vex8ABx10EAcddNBmw6dPn86rXvUqLr/8cubMmbPZa950002bpt1///2ZNWsWK1euHHaewy2XVKQStrFNLcPWpu+m/tHYMDd3fW624JU6ZckSaNlAANXzJUuGHn8sXn8s4+j08o9HDz88uvbtsPPOO296PHHiRJ5//vlNz1/+8pcD1ZE/b3/721m+fDnLly/nvvvu40tf+hKTJk3i9ttv5+STT+bqq6/m2GOPHXK+1Y+s7XPKKafwwQ9+cLNDprbHcMslFamEbWxTy7BkCey000vbJ0/urv7R2DA3D6mbcrMFr9QpixbB0qUwa1Z1hchZs6rnixaN/esDTJxY3Y9VHJ1e/vGogb0Z06ZN46mnnhrVNIcddhg333wzq1atAuBXv/oVK1eu5Omnn+YXv/gF73rXu/jsZz/LXXfdtdX5HHnkkVxxxRVs3LiRdevWccMNNzB//vzNxnnLW97C1772NQDuueceVqxY8ZL5nHjiifzFX/wFxxxzzEvm31+fR7Vy5UoefvhhXv/61w87z+GWSypSCdvYppZh0SK45BKYPv3FtunT4eKLu6t/NDbMzV2fmye1dW6SRmfRos4m115//fFmyZLqCqSth07t4N6M6dOnc8QRR3DAAQfwspe9jN133/bfps6YMYNLL72UhQsX8utf/xqAj33sY0ybNo3jjz+eZ599lszkM5/5zFbnc+KJJ3LLLbfwxje+kYjgU5/6FK95zWs2O1TrrLPO4owzzmDOnDnMmTOHQw899CXzmTZtGueee+5L2s8++2zOOussDjzwQCZNmsSll17KzjvvPOw8h1uu/fbbb5t9InWlEraxTS1DCX2jsWFu7vrcHO3evT0e9fX1ZY/8v6Kkceb+++9nzpw5I5+gv786L+jhh6tfj5cs8UtZhw31HkbEHZnZ16GQimBultQp5ubuN5rc7B5eSRpP3OsgSdL4Ym7uap7DK0mSJEkqkgWvJEmSJKlIFryS1LBeuFZCqXzvJKlMbt+712jfOwteSWrQlClTWL9+vYm1C2Um69evZ8qUKZ0ORZLURubm7rU9udmLVklSg2bOnMnatWtZt25dp0PRdpgyZQozZ87sdBiSpDYyN3e30eZmC15JatBOO+3E3nvv3ekwJElSzdzcWzykWZIkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFanRgjcijo2IByJiVUScN8TwWRFxbUSsiIjrI2Jm3T4vIm6JiHvrYae0THNpRPwkIpbXt3lNLoMkSSUxN0uSekljBW9ETAQ+D7wTmAssjIi5W4x2EfDlzDwIuBD4eN2+AfhAZr4BOBb4u4jYtWW6j2bmvPq2vKllkCSpJOZmSVKvaXIP73xgVWY+lJnPAZcDx28xzlzgB/Xj6waHZ+bKzPxx/fgx4GfAjAZjlSSpF5ibJUk9pcmC97XAIy3P19Ztre4CTqofnwhMi4jprSNExHxgMvBgS/OS+nCqz0bEzu0NW5KkYpmbJUk9pdMXrToHWBARdwILgEeBjYMDI2IP4CvAGZn5Qt18PrA/8Cbgt4Bzh5pxRCyOiIGIGFi3bl2DiyBJUlHMzZKkYjRZ8D4K7NnyfGbdtklmPpaZJ2XmwcAFdduTABGxC/Bd4ILMvLVlmsez8mvgEqrDs14iM5dmZl9m9s2Y4RFXkiRhbpYk9ZgmC95lwL4RsXdETAZOBa5qHSEidouIwRjOBy6u2ycD36K6aMaVW0yzR30fwAnAPQ0ugyRJJTE3S5J6SmMFb2Y+D3wIuAa4H/hGZt4bERdGxHH1aEcBD0TESmB3YEnd/l7gLcDpQ/zFQX9E3A3cDewGfKypZZAkqSTmZklSr4nM7HQMjevr68uBgYFOhyFJKkRE3JGZfZ2Oo5uZmyVJ7TRcbu70RaskSZIkSWqEBa8kSZIkqUgWvJIkSZKkIlnwSpIkSZKKZMErSZIkSSqSBa8kSZIkqUgWvJIkSZKkIlnwSpIkSZKKZMErSZIkSSqSBa8kSZIkqUgWvJIkSZKkIlnwSpIkSZKKZMErqbv198Ps2TBhQnXf39/piCRJJTC/SEWY1OkAJGm79ffD4sWwYUP1fM2a6jnAokWdi0uS1N3ML1Ix3MMrqXtdcMGLX0YGbdhQtUuStL3ML1IxLHglda+HHx5duyRJI2F+kYphwSupe+211+jaJUkaCfOLVAwLXknda8kSmDp187apU6t2SZK2l/lFKoYFr6TutWgRLF0Ks2ZBRHW/dKkXFJEk7Rjzi1QMr9IsqbstWuQXEElS+5lfpCK4h1eSJEmSVCQLXkmSJElSkSx4JUmSJElFsuCVJEmSJBXJgleSJEmSVCQLXkmSJElSkSx4JUmSJElFsuCVJEmSJBXJgleV/n6YPRsmTKju+/s7HZEkSZK0dX6H1TZM6nQAGgf6+2HxYtiwoXq+Zk31HGDRos7FJUmSJA3H77AaAffwCi644MUNxaANG6p2SZIkaTzyO6xGwIJX8PDDo2uXJEmSOs3vsBoBC17BXnuNrl2SJEnqNL/DagQseAVLlsDUqZu3TZ1atUuSJEnjkd9hNQIWvKpO6l+6FGbNgojqfulST/aXJEnS+OV3WI2AV2lWZdEiNw6SJEnqLn6H1TY0uoc3Io6NiAciYlVEnDfE8FkRcW1ErIiI6yNiZt0+LyJuiYh762GntEyzd0TcVs/zioiY3OQySJJUEnOzJKmXNFbwRsRE4PPAO4G5wMKImLvFaBcBX87Mg4ALgY/X7RuAD2TmG4Bjgb+LiF3rYZ8EPpuZrwOeAM5sahkkSSqJuVmS1Gua3MM7H1iVmQ9l5nPA5cDxW4wzF/hB/fi6weGZuTIzf1w/fgz4GTAjIgI4GriynuYy4IQGl0GSpJKYmyVJPaXJgve1wCMtz9fWba3uAk6qH58ITIuI6a0jRMR8YDLwIDAdeDIzn9/KPCVJ0tDMzZKkntLpqzSfAyyIiDuBBcCjwMbBgRGxB/AV4IzMfGE0M46IxRExEBED69ata2fMkiSVzNwsSSpGkwXvo8CeLc9n1m2bZOZjmXlSZh4MXFC3PQkQEbsA3wUuyMxb60nWA7tGxKTh5tky76WZ2ZeZfTNmzGjTIkmS1NXMzZKkntJkwbsM2Le+cuNk4FTgqtYRImK3iBiM4Xzg4rp9MvAtqotmDJ4TRGYm1flEJ9dNpwHfaXAZJEkqiblZktRTGit463N5PgRcA9wPfCMz742ICyPiuHq0o4AHImIlsDuwpG5/L/AW4PSIWF7f5tXDzgU+EhGrqM4b+lJTyyBJUknMzZKkXhPVD7Nl6+vry4GBgU6HIUkqRETckZl9nY6jm5mbJUntNFxu7vRFqyRJkiRJaoQFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSJNGOmJEvBqYMvg8Mx9uJCJJkjQi5mZJkrZum3t4I+K4iPgx8BPgP4DVwL81HJckSRqGuVmSpJEZySHN/wM4DFiZmXsDbwVubTQqSZK0NeZmSZJGYCQF728ycz0wISImZOZ1QF/DcUmSpOGZmyVJGoGRnMP7ZES8ArgB6I+InwG/ajYsSZK0FeZmSZJGYCR7eI8HNgD/Hfge8CDw7iaDkiRJW2VuliRpBEZS8P5lZr6Qmc9n5mWZ+Tng3KYDkyRJwzI3S5I0AiMpeN8+RNs72x2IJEkaMXOzJEkjMOw5vBFxFnA2sE9ErGgZNA24uenAJEnS5szNkiSNztb28H4N+H3gqvp+8HZoZr5vDGIb//r7YfZsmDChuu/v73REksaCn311jrl5kJ9DjSXXN6lrDVvwZuYvMnN1Zi4E9gSOzsw1VH+BsPeYRThe9ffD4sWwZg1kVveLF7sBlErnZ18dZG6u+TnUWHJ9k7paZObWR4j4K6r/9nt9Zu4XEb8NfDMzjxiLANuhr68vBwYG2jvT2bOrDd6WZs2C1avb+1qSxg8/+wIi4o7M7Nj/3vZ8bvZzqLHk+iZ1heFy80guWnUicBz1//tl5mNU5wr1tocfHl27pDL42df40Nu52c+hxpLrm9TVRlLwPpfVbuAEiIiXNxtSl9hrr9G1SyqDn32ND72dm/0caiy5vkldbSQF7zci4h+BXSPij4H/DXyx2bC6wJIlMHXq5m1Tp1btksrlZ1/jQ2/nZj+HGkuub1JX22bBm5kXAVcC/wzsR/Vn938/kplHxLER8UBErIqI84YYPisiro2IFRFxfUTMbBn2vYh4MiKu3mKaSyPiJxGxvL7NG0ksbbdoESxdWp2/EVHdL11atUsql599jQM9n5v9HGosub5JXW2bF60CiIjXAPOpDp1alpk/HcE0E4GVwNuBtcAyYGFm3tcyzjeBqzPzsog4GjgjM99fD3srMBX4k8x8d8s0l9bTXDnShWzkolWSpJ7V6YtW1TGYmyVJqm33Rasi4o+A24GTgJOBWyPiD0fwmvOBVZn5UGY+B1wOHL/FOHOBH9SPr2sdnpnXAk+N4HUkSeop5mZJkkZmJOfwfhQ4ODNPz8zTgEOBc0cw3WuBR1qer63bWt1FlayhuuLktIiYPoJ5L6kPtfpsROw8gvElSSqJuVmSpBEYScG7ns1/zX2qbmuHc4AFEXEnsAB4FNi4jWnOB/YH3gT8FsMk+IhYHBEDETGwbt26NoUrSdK4YG6WJGkEJg03ICI+Uj9cBdwWEd+hOk/oeGDFCOb9KLBny/OZddsm9f8GnlS/3iuAP8jMJ7c208x8vH7464i4hCoxDzXeUmApVOcJjSBeSZLGNXOzJEmjs7U9vAdT/Yn97wPfpv6vP+A7wE9GMO9lwL4RsXdETAZOBa5qHSEidouIwRjOBy7e1kwjYo/6PoATgHtGEIskSSUwN0uSNArD7uEFDqE6JOkPgBH91UGrzHw+Ij4EXANMBC7OzHsj4kJgIDOvAo4CPh4RCdwAfHBw+oi4kerwqFdExFrgzMy8BuiPiBlAAMuBPx1tbJIkdSlzsyRJozDs3xJFxIeBs4C9gcdaBwGZmfs0H157+NcHkqR26tTfEpmbJUka2qj/ligzP5eZc4BLMnOfltve3ZRQJUkqhblZkqTR2eZVmjPzrLEIRJIkjYy5WZKkkRnJ3xJJkiRJktR1LHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUVqtOCNiGMj4oGIWBUR5w0xfFZEXBsRKyLi+oiY2TLsexHxZERcvcU0e0fEbfU8r4iIyU0ugyRJJTE3S5J6SWMFb0RMBD4PvBOYCyyMiLlbjHYR8OXMPAi4EPh4y7BPA+8fYtafBD6bma8DngDObHfskiSVyNwsSeo1Te7hnQ+sysyHMvM54HLg+C3GmQv8oH58XevwzLwWeKp15IgI4GjgyrrpMuCEtkcuSVKZzM2SpJ7SZMH7WuCRludr67ZWdwEn1Y9PBKZFxPStzHM68GRmPr+VeUqSpKGZmyVJPaXTF606B1gQEXcCC4BHgY3tmHFELI6IgYgYWLduXTtmKUlSLzA3S5KK0WTB+yiwZ8vzmXXbJpn5WGaelJkHAxfUbU9uZZ7rgV0jYtJw82yZ99LM7MvMvhkzZmznIkiSVBRzsySppzRZ8C4D9q2v3DgZOBW4qnWEiNgtIgZjOB+4eGszzMykOp/o5LrpNOA7bY1akqRymZslST2lsYK3PpfnQ8A1wP3ANzLz3oi4MCKOq0c7CnggIlYCuwNLBqePiBuBbwJvjYi1EXFMPehc4CMRsYrqvKEvNbUMkiSVxNwsSeo1Uf0wW7a+vr4cGBjodBiSpEJExB2Z2dfpOLqZuVmS1E7D5eZOX7RKkiRJkqRGWPBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwTsS/f0wezZMmFDd9/d3OiKNV92wrnRDjJLU69xWd5b9/yL7Ql1uUqcDGPf6+2HxYtiwoXq+Zk31HGDRos7FpfGnG9aVbohRknqd2+rOsv9fZF+oAJGZnY6hcX19fTkwMLB9E8+eXX24tzRrFqxevSNhqTTdsK50Q4xSF4iIOzKzr9NxdLMdys2lc1vdWfb/i+wLdZHhcrOHNG/Lww+Prl29qxvWlW6IUZJ6ndvqzrL/X2RfqAAWvNuy116ja1fv6oZ1pRtilKRe57a6s+z/F9kXKoAF77YsWQJTp27eNnVq1S616oZ1pRtilKRe57a6s+z/F9kXKoAF77YsWgRLl1bnKkRU90uXeqK+Xqob1pVuiFGSep3b6s6y/19kX6gAXrRKkqRR8qJVO87cLElqJy9aJUmSJEnqKRa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkorUaMEbEcdGxAMRsSoizhti+KyIuDYiVkTE9RExs2XYaRHx4/p2Wkv79fU8l9e3Vze5DJIklcTcLEnqJZOamnFETAQ+D7wdWAssi4irMvO+ltEuAr6cmZdFxNHAx4H3R8RvAX8F9AEJ3FFP+0Q93aLMHGgqdkmSSmRuliT1mib38M4HVmXmQ5n5HHA5cPwW48wFflA/vq5l+DHA9zPz/9aJ9PvAsQ3GKklSLzA3S5J6SpMF72uBR1qer63bWt0FnFQ/PhGYFhHTRzDtJfUhU/9fRER7w5YkqVjmZklST+n0RavOARZExJ3AAuBRYOM2plmUmQcCR9a39w81UkQsjoiBiBhYt25dO2OWJKlk5mZJUjGaLHgfBfZseT6zbtskMx/LzJMy82Dggrrtya1Nm5mD908BX6M6POslMnNpZvZlZt+MGTPaskCSJHU5c7Mkqac0WfAuA/aNiL0jYjJwKnBV6wgRsVtEDMZwPnBx/fga4B0R8aqIeBXwDuCaiJgUEbvV0+4EvBu4p8FlkCSpJOZmSVJPaazgzczngQ9RJcj7gW9k5r0RcWFEHFePdhTwQESsBHYHltTT/l/gf1Al5mXAhXXbzlTJdQWwnOqX5S82tQySJJXE3CxJ6jWRmZ2OoXF9fX05MOA/JUiS2iMi7sjMvk7H0c3MzZKkdhouN3f6olWSJEmSJDXCgleSJEmSVCQLXkmSJElSkSx4JUmSJElFsuCVJEmSJBXJgleSJEmSVCQLXkmSJElSkSx4JUmSJElFsuCVJEmSJBXJgleSJEmSVCQLXkmSJElSkSx4JUmSJElFsuCVJEmSJBXJgleSJEmSVCQLXkmSJElSkSx4JUmSJElFsuDtNf39MHs2TJhQ3ff3dzqisVdiH4zHZRoqpibjHI99ULot+/zss8fne+C6IalJ7d7GDM4vAiZNqu6b2na5fewO23qfticfd/q9H8vXz8zib4ceemgqM7/61cypUzPhxdvUqVV7ryixD8bjMg0V0047ZU6e3Eyc47EPSjdUn295Gw/vQUPrBjCQ4yC/dfPN3KwitHsbs7Vta7u3qebO7rCt92l78nGn3/sxzs1RDStbX19fDgwMdDqMzps9G9aseWn7rFmwevVYR9MZJfbBeFym4WIaSjviHI99ULqRvsedfg8aWjci4o7M7NvuGcjcrDK0exuzrW1rO7ep5s7usK33aXvycaff+zHOzRa8vWTChOo3lC1FwAsvjH08nVBiH4zHZRoupqG0I87x2AelG+l73On3oKF1w4J3x5mbVYR2b2O2tW1t5zbV3NkdtvU+bU8+7vR7P8a52XN4e8lee42uvUQl9sF4XKbRvHY74hyPfVC6kfZtp98D1w1JTWr3NmZb07Vz2+X2sTts633annzc6fd+jF/fgreXLFkCU6du3jZ1atXeK0rsg/G4TEPFtNNOMHny5m3tinM89kHphurzLY2H98B1Q1KT2r2N2dq2td3bLreP3WFb79P25ONOv/dj/fpDndhb2s0LY7T46lczZ83KjKjue/HCBCX2wXhcpqFiajLO8dgHpduyz886a3y+Bw2sG3jRKnOzNKjd25jB+UHmxInVfVPbVHNnd9jW+7Q9+bjT7/0Y5mbP4ZUkaZQ8h3fHmZslSe3kObySJEmSpJ5iwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiWfBKkiRJkopkwStJkiRJKpIFryRJkiSpSBa8kiRJkqQiRWZ2OobGRcQ6YE2n4yjUbsDPOx1E4ezj5tnHzSqxf2dl5oxOB9HNxmlu7uZ11dg7w9g7w9g7Y7zHPmRu7omCV82JiIHM7Ot0HCWzj5tnHzfL/lW36OZ11dg7w9g7w9g7o1tj95BmSZIkSVKRLHglSZIkSUWy4NWOWtrpAHqAfdw8+7hZ9q+6RTevq8beGcbeGcbeGV0Zu+fwSpIkSZKK5B5eSZIkSVKRLHi1SUQcGxEPRMSqiDhviOF7RcR1EXFnRKyIiHfV7YsiYnnL7YWImFcPOzQi7q7n+bmIiDFerHGloT5eEhGPRMTTY7w441K7+zgipkbEdyPiRxFxb0R8YuyXanxpaD3+XkTcVffx/4qIiWO8WCrQttbVepz3RsR99br3tZb270XEkxFx9Rbj39iyDj8WEd/uotjfGhE/rGO/KSJe10WxH13Hfk9EXBYRk8ZT7HW+uKVuWxERp7SMv3dE3FbP84qImNwlcX+onl9GxG7tjrnh2Pvred4TERdHxE5dFPuX6ny4IiKujIhXdEvsLdN9LsbT99LM9OYNYCLwILAPMBm4C5i7xThLgbPqx3OB1UPM50DgwZbntwOHAQH8G/DOTi9rgX18GLAH8HSnl7HTtyb6GJgK/F79eDJwo+txI+vxLvV9AP8MnNrpZfXW3bcRrqv7AncCr6qfv7pl2FuB3weu3spr/DPwgW6JHVgJzKkfnw1c2g2xU+2geQTYr35+IXDmeIod2A/Yt37828DjwK71828MbtOA/zW4feyCuA8GZgOrgd3a3d8Nx/4uqnwSwNfb3ecNx75Ly/SfAc7rltjrtj7gK4yj76Xu4dWg+cCqzHwoM58DLgeO32KcBHapH78SeGyI+SyspyUi9qD60N6a1Sfgy8AJDcTeLdrexwB1/z7eQLzdqO19nJkbMvO6+vFzwA+BmQ3E3i2aWo9/WT+cRJV8vcCEdtRI1tU/Bj6fmU8AZObPBgdk5rXAU8PNPCJ2AY4Gvt3muKG52Efy2dxRTcQ+HXguM1fWz78P/MF4ij0zV2bmj+vHjwE/A2ZERFCtJ1fW019G+78LtT3u+vmdmbm6zbGOVez/mjWqnS9N5O2mYv8lQL3uvIxm8mEjsUd1dNangb9oIObtZsGrQa+l+vV00Nq6rdVfA++LiLXAvwL/bYj5nEL1S9rgPNduY569pIk+1uYa7eOI2JVqz8O1bYi1WzXWxxFxDVXifIoXvxxK22sk6+p+wH4RcXNE3BoRx45i/icA17b8WNNOTcX+R8C/1p/N9wNNnKLRROw/ByZFRF/9/GRgz7ZEu7m2xB4R86l+uHuQqlh/MjOf38o8x2PcY6XR2OtDmd8PfK+tUVcaiz0iLgF+CuwP/H27A6e52D8EXDXedsRY8Go0FlId/jST6lCRr0TEpnUoIv4LsCEz7+lUgAWwj5u3XX0c1fliXwc+l5kPjWXAXWi7+jgzj6E6PH9nqj0iUtMmUR22dxTVevvF+oetkVhIZ3983J7Y/zvwrvqzeQnV4ZKdMKrY6710pwKfjYjbqX4U29h8mEPaauz10W1fAc7IzBc6EeAwujVu2LHYvwDckJk3jk2oL7FdsWfmGVSHC99P9QNxJ4wq9oj4beA9NFOg7xALXg16lM1/LZ1Zt7U6k+o8FDLzFmAK0HoRg1PZPPk/yuaHkAw1z17SRB9rc0328VLgx5n5d+0Ktks1uh5n5rPAd3jpoVXSaI1kXV1LtTfiN5n5E6pzXPfd1ozrC/jMB77bpli31PbYI2IG8MbMvK1uugJ4c/tC3qSRfs/MWzLzyMycD9xQT9NuOxR7fZj7d4ELMvPWevz1wK7x4kW2mvgu1ETcY6Wx2CPir6gOtf1It8UOkJkbqQ41buLw/SZiPxh4HbAqIlYDUyNiVQOxj5oFrwYtA/aN6kqCk6m+kF61xTgPU11MgoiYQ/Uldl39fALwXjY/J+9x4JcRcVh9HsIHqL7I9qq297FeopE+joiPUZ3v9udNBt8l2t7HEfGK+pfiwT3p/xX4UcPLofKNZF39NtXei8Eidj9gJEdwnEx1UaVn2xbt5pqI/QnglRGxX/387VR7j9qtkX6PiFfX9zsD51Jd/Kndtjv2evxvAV/OzE2nZNR7p6+jWmcATqP934XaHvcYaiT2iPgj4BhgYYN7rNsee1ReN/gYOI5m8mET6/p3M/M1mTk7M2dTHcnVyJXgRy3HwZWzvI2PG9WhhyupjsO/oG67EDiufjwXuJnqSm7LgXe0THsUcOsQ8+wD7qnn+Q9AdHo5C+zjT1H9CvdCff/XnV7OkvqY6lfPpPpiuLy+/VGnl7OwPt6dKvmuqLcXfw9M6vRyeuv+2wjW1aA6rPc+4G5arg5OdUX2dcAz9bb1mJZh1wPHdlvswIn1uHfVy7BPF8X+6Xo7/ADw5+Ot34H3Ab9pyRPLgXn1sH2oLpy0CvgmsHOXxP3h+j14nuoCZ//URX3+fD2/wfa/7IbYqXZG3lyPew/QT8tVm8dz7EPMf9xcpTnqgCRJkiRJKoqHNEuSJEmSimTBK0mSJEkqkgWvJEmSJKlIFrySJEmSpCJZ8EqSJEmSimTBK0mSJPWQiOiLiM9tZfhvR0Qn/pdXajv/lkiSJEnaARExMTM39urrS+OZe3ilHhIR346IOyLi3ohYXLc93TL85Ii4tH68e0R8KyLuqm9v7lDYkiS1zTC58E8j4tMt45weEf9QP35fRNweEcsj4h8jYmLd/nRE/G1E3AUcHhF/GRHLIuKeiFgaEVGP96aIWFFP/+mIuKdun1g/X1YP/5MhYp0dET+KiP6IuD8iroyIqfWw1RHxyYj4IfCeiHhHRNwSET+MiG9GxCtaXv//1Ln89oiYFhFHRcTV9fAFdWzLI+LOevjsljinRMQlEXF3Pfz3WvroXyLiexHx44j4VFPvmbQjLHil3vKHmXko0Ad8OCKmb2XczwH/kZlvBA4B7h2LACVJathQufCfgRNbxjkFuDwi5tSPj8jMecBGYFE9zsuB2zLzjZl5E/APmfmmzDwAeBnw7nq8S4A/aZl+0JnALzLzTcCbgD+OiL2HiPf1wBcycw7wS+DslmHrM/MQ4H8D/y/wtvr5APCRiJgMXAH8WZ3P3wY8s8X8zwE+WMd35BDDPwhkZh4ILAQui4gp9bB5df8cCJwSEXsOEb/UURa8Um/5cP1L9K3AnsC+Wxn3aOB/AmTmxsz8xRjEJ0lS016SCzNzHfBQRBxWF8D7AzcDbwUOBZZFxPL6+T71fDZSFcqDfi8ibouIu6ly6BsiYldgWmbeUo/ztZbx3wF8oJ7vbcB0hs7Lj2TmzfXjrwK/2zLsivr+MGAucHM9v9OAWVTF8uOZuQwgM3+Zmc9vMf+bgc9ExIeBXYcY/rv165KZPwLWAPvVw67NzF9k5rPAffVrSuPKpE4HIGlsRMRRVL/sHp6ZGyLiemAK0Hoi/5SXTilJUhm2kgsBLgfeC/wI+FZmZn1Y8mWZef4Qs3t28LzZeo/nF4C+zHwkIv6abefUAP5bZl6zjfG2vOBO6/Nftczr+5m5cLMXiDhwG/MmMz8REd8F3kVVMB8DPLut6Wq/bnm8EWsLjUPu4ZV6xyuBJ+oEvz/Vr8EA/xkRcyJiApsfznUtcBZsOs/olWMbriRJbTdcLgT4FnA81WG7l9dt1wInR8SrASLityJiqL2Yg8Xtz+tzZ08GyMwngaci4r/Uw09tmeYa4KyI2Kme934R8fIh5r1XRBxeP/5/gJuGGOdW4IiIeF09r5dHxH7AA8AeEfGmun1aRGxWlEbE72Tm3Zn5SWAZ1d7tVjdSH8Zdz3Over5SV7DglXrH94BJEXE/8Amq5AhwHnA18H+Ax1vG/zOqw7PuBu6gOlRKkqRuNlwuJDOfAO4HZmXm7XXbfVTnxv57RKwAvg/sseVM68L2i8A9VIXsspbBZwJfrA81fjkweIrQP1EdBvzD+gJR/8jQe0gfAD5Yx/wq6tONtnj9dcDpwNfrOG8B9s/M56jOsf37+jDu7/PSPc9/Xl9oawXwG+Dfthj+BWBC/X3gCuD0zPw1Upfwb4kkSZKkhkTEKzLz6frxecAemflnI5x2NnB1fSEsSdvB4+wlSZKk5vzXiDif6nv3Gqo9sZLGiHt4JUmSJElF8hxeSZIkSVKRLHglSZIkSUWy4JUkSZIkFcmCV5IkSZJUJAteSZIkSVKRLHglSZIkSUX6/wE5AYBkILpGUAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PLOT ThresholdClassifier SEARCH RESULTS ###\n",
    "\n",
    "plt.figure(figsize=(16,5))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.scatter(\n",
    "    threshold_model_test_roc_auc,\n",
    "    threshold_model_test_fbeta,\n",
    "    c='red', label='thresholdModel'\n",
    ")\n",
    "plt.ylabel('fbeta'); plt.xlabel('auc'); plt.legend()\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(\n",
    "    threshold_model_test_avg_precision,\n",
    "    threshold_model_test_fbeta,\n",
    "    c='red', label='thresholdModel'\n",
    ")\n",
    "plt.ylabel('fbeta'); plt.xlabel('average precision'); plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "641f305d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:36:48.719851Z",
     "iopub.status.busy": "2023-02-19T19:36:48.719490Z",
     "iopub.status.idle": "2023-02-19T19:36:49.006000Z",
     "shell.execute_reply": "2023-02-19T19:36:49.004108Z"
    },
    "papermill": {
     "duration": 0.294064,
     "end_time": "2023-02-19T19:36:49.008067",
     "exception": false,
     "start_time": "2023-02-19T19:36:48.714003",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwgAAAFVCAYAAABLisgXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo90lEQVR4nO3de7zUdZ348ddbvJBK6iJphQnuasEPUAlQw9RWTWtdzSKDdCut3DRzdyvLX7VdzHal7Werrtp11cwrWl7KVct08R4oiBfETFGxzQVSE8X7+/fHDN+GA8w52pmZz3Bez8fjPJj5zpeZ15k5HM77fC8TmYkkSZIkAazT6QBJkiRJ5XBAkCRJklRxQJAkSZJUcUCQJEmSVHFAkCRJklRZt9MBr9Tmm2+eI0aM6HSGJEmS1LVuu+22JZk5bHW3dd2AMGLECGbPnt3pDEmSJKlrRcRDa7rNXYwkSZIkVRwQJEmSJFUcECRJkiRVuu4YhNV54YUXWLRoEc8++2ynU/QqDB48mOHDh7Peeut1OkWSJGnAWysGhEWLFjFkyBBGjBhBRHQ6R69AZrJ06VIWLVrEyJEjO50jSZI04K0Vuxg9++yzDB061OGgC0UEQ4cOdeuPJElSIdaKAQFwOOhivnaSJEnlWGsGBEmSJEl/vrVyQIjo34++WLhwIWPGjGnJ53PdddcREfzgBz+ols2dO5eI4Fvf+laf76cvja38PCRJklS+tXJAWBuNGTOGCy+8sLp+3nnnsf3223ewSJIkSWsjB4R+9OKLL3LwwQczatQopkyZwjPPPMNxxx3HxIkTGTNmDIcffjiZCcDJJ5/M6NGjGTduHFOnTgXg6aef5rDDDmPSpEnsuOOOXHrppdV9b7311jz77LM89thjZCZXXnkl73rXu6rb586dy84778y4ceM48MADefzxxwG47bbb2H777dl+++059dRTq/VfeukljjnmGCZOnMi4ceP47ne/246nSJIkSYVzQOhHCxYs4Mgjj2T+/Pm89rWv5bTTTuOoo45i1qxZ3HXXXSxfvpyf/exnAJxwwgnMmTOHefPm8Z3vfAdmz+YbRx3FX48Ywa9PO41rTzyRY44+mqevvx4WLIAnn2TKpEnMmDGDm266ifHjx7PBBhtUj/2hD32I6dOnM2/ePMaOHcvXvvY1AA499FBOOeUU7rjjjpVaf/jDH7LJJpswa9YsZs2axfe//30efPDB9j1Za4P+2j9N7eNrJkmr8nujenBA6EdbbbUVkydPBuCQQw7hhhtu4Nprr2WnnXZi7Nix/OpXv+Luu+8GYNy4cRx88MH8+Mc/Zt11a29HcfWtt3LCWWexwwc/yB5///c8+9xzPPz731f3f9BeezFjxgzOO+88pk2bVi1/8skneeKJJ9h9990B+PCHP8zMmTN54okneOKJJ9htt90A+Lu/+7vq71x99dX86Ec/YocddmCnnXZi6dKl/OY3v2ntEyRJkqTirRVvlFaKnqfrjAiOPPJIZs+ezVZbbcVXv/rV6nz/P//5z5k5cyaXX3453/jGN7jzjDPITC6ePp03jxix0v089oc/ALDl5puz3nrr8Ytf/IKTTjqJm2666VW3ZiannHIK++yzz0rLFy5c+KrvU5Ikaa3Tly0o9V3I1xZuQehHDz/8MDfffDMA5557LrvuuisAm2++OcuWLeOiiy4C4OWXX+aRRx7hHe94B9OnT+fJJ59k2fLl7LPzzpxy4YXVcQpzFixY5TGOO+44pk+fzqBBg6plm2yyCZttthnXX389AGeffTa77747m266KZtuuik33HADAOecc071d/bZZx9OP/10XnjhBQDuu+8+nn766f5+SvqXm0AlSZJabq3cgtCpIe7Nb34zp556KocddhijR4/miCOO4PHHH2fMmDFsueWWTJw4EagdIHzIIYfw5JNPkpkcffTRbDpkCP/80Y/yjyeeyLhp03j55ZcZ+cY38rNvf3ulx3jb29622sc+66yz+MQnPsEzzzzDNttswxlnnAHAGWecwWGHHUZE8M53vrNa/2Mf+xgLFy5k/PjxZCbDhg3jkksuac0TI0mv1gD8zV3X8zWTul5kl/0jnTBhQs6ePXulZfPnz2fUqFEdKuonPT6nNZowobUdHdKn17C0/3RK61HvfM26j69Z9/E16z6+Zs2tpc9PRNyWmav9wXKt3IKgftKXoWUtHVi61lr6TUwqhv/GJA0AHoMgSZIkqTJwtyD423Gp9fxta3MlPj8lNmnN+npyhoH8mpX2NV1aT4l8jpprw/PjFgRJkiRJFQcESZIkSRUHBEmSJEmVtXNA6Msbak2c2PtHH99864knnuC0004D4LrrrmO//fbr90/pI1/9Khddc02f11+4cCFjxoxZ7W177LEHK04VO2LECN7+9revdPsOO+ywxr+7Jo33+eesI0mSpM5aOweENmscEPrqpZdealHNK/fUU0/xyCOPALX3I5AkSdLA5YDQD4499lh++9vfssMOO3DMMcewbNkypkyZwlve8hYOPvhgVrwZ3YgRI/j85z/P+PHjmTFjBldffTW77LIL48eP5/3HHsuyZ56p3d8ppzD6oIMYN20an/33f68eZ+acObztbW9jm2224aKLLgIgMznmmGMYM2YMY8eO5YILLlilb/ny5UydOpVRo0Zx4IEHsnz58pVuP+igg6q/d9555zFt2rTqtmefe45Dv/Y1xk6dyo4HH8y19S0Ay599lqlf+MJq73Olz+v972fZsmX98CxLkiSpHQbuaU770QknnMBdd93F3Llzue666zjggAO4++67ecMb3sDkyZO58cYb2XXXXQEYOnQot99+O0uWLOG9730vv/zlL9loo42Y/qlPceI55/DJ97+fn153HfdedBERwRNPPVU9zv8sWcINN9zAvffey/7778+UKVP4yU9+wty5c7njjjtYsmQJEydOZLfddlup7/TTT2fDDTdk/vz5zJs3j/Hjx690+/ve9z4OPfRQPvvZz3L55ZdzzjnncPbZZwNw6owZRAR3nn8+9y5cyDuPOor7Lr6Y0y++mA0HD17lPpcsWcLxxx//p89r+nROPPFEvvzlL7fyJZAkSVI/cUBogUmTJjF8+HCgtj//woULqwHhAx/4AAC33HIL99xzD5MnTwbg+T/+kV3GjmWTjTdm8AYb8NGvf539dt2V/RqOD3jP7ruzzjrrMHr0aB577DEAbrjhBqZNm8agQYPYYost2H333Zk1axbjxo2r/t7MmTM5+uijARg3btxKt0FtaNlss804//zzGTVqFBtuuGF12w133MGnDjoIgLeMGMHWr3899z38MDPnzOHo+ufSeJ+rfF7PP88uu+zSH0+rJEmS2sABoQU22GCD6vKgQYN48cUXq+sbbbQRUNs1aO+99+a8886r3dBw8O6vzzyTa2bN4qJrruE/ZszgV6efXrvf9dev1sl+foOQD3zgA3zyk5/kzDPP/LPuZ5XPS5IkSV3FYxD6wZAhQ3iqYVegvth555258cYbuf/++wF4evly7nvoIZY98wxPLlvGuydP5tuf/jR3/OY3Te/n7W9/OxdccAEvvfQSixcvZubMmUyaNGmldXbbbTfOPfdcAO666y7mzZu3yv0ceOCBfO5zn2OfffZZ+f532IFzrrwSgPseeoiHf/973rz11uy2446ce9VVq9znKp/X009z3333vaLnRpIkSZ2zdm5B6Mtv1/tyus0JE/r0cEOHDmXy5MmMGTOG17zmNWyxxRa9/p1hw4Zx5plnMm3aNJ577jlYvpzjP/EJhmy0EQd85jM8+/zzZCYn/uM/Nr2fAw88kJtvvpntt9+eiOCb3/wmW265JQsXLqzWOeKIIzj00EMZNWoUo0aN4q1vfesq9zNkyBA+//nPr7L8yClTOOKEExg7dSrrDhrEmV/5Chusvz5HvO99HHrccavc5yqfF3D88cez3Xbb9fqcSJIkqfOiv3dVabUJEyZkz3Ppz58/n1GjRr2yO+rHAaFf9PX9AUpr6qeePr2GfXhPij4Nh/2ltB4or8me5krrgfKa7GmuLz1QXpM9zfnvvvd17GmuDz0RcVtmrvYHOXcxkiRJklRxQJAkSZJUWWsGhG7bVUp/4msnSZJUjrViQBg8eDBLly71B80ulJksXbqUwYMHdzpFkiRJrCVnMRo+fDiLFi1i8eLFff9LS5b0vs78+a8+6pXqSw+U19QPPYMHD67eWE6SOqVPx/21PkPqN30+hr21GepCa8WAsN566zFy5MhX9pdGj+59nXZukehLD5TX5FYbSZKktcpaMSD05G+Buo+vmSRJUhnWygGhNCX+8FtiU0l8frqPr5kkSf1jrThIWZIkSVL/cAuCJEnqE7fUNefz0zufo+7ggCDpVfGbfPcp7TUrrac0Pj+98znSn8uvodVzFyNJkiRJlZYOCBGxb0QsiIj7I+LY1dz+poi4NiLmRMS8iHh3K3skSZIkNdeyXYwiYhBwKrA3sAiYFRGXZeY9Dat9CbgwM0+PiNHAFcCIVjVJ3co3u+k+braWJL1Spfzf0cotCJOA+zPzgcx8HjgfOKDHOgm8tn55E+B3LeyRJEmS1ItWDghvBB5puL6ovqzRV4FDImIRta0Hn1rdHUXE4RExOyJmL168uBWtkiRJkuj8QcrTgDMzczjwbuDsiFilKTO/l5kTMnPCsGHD2h4pSZIkDRStHBAeBbZquD68vqzRR4ELATLzZmAwsHkLmyRJkiQ10coBYRawbUSMjIj1ganAZT3WeRjYEyAiRlEbENyHSJIkSeqQlg0ImfkicBRwFTCf2tmK7o6I4yJi//pqnwE+HhF3AOcBH8lMT+whSZIkdUhL30k5M6+gdvBx47IvN1y+B5jcygZJkiRJfdfpg5QlSZIkFcQBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVLFAUGSJElSxQFBkiRJUsUBQZIkSVKlpQNCROwbEQsi4v6IOHYN6xwUEfdExN0RcW4reyRJkiQ1t26r7jgiBgGnAnsDi4BZEXFZZt7TsM62wP8FJmfm4xHxulb1SJIkSepdK7cgTALuz8wHMvN54HzggB7rfBw4NTMfB8jM/21hjyRJkqRetHJAeCPwSMP1RfVljbYDtouIGyPilojYd3V3FBGHR8TsiJi9ePHiFuVKkiRJ6vRByusC2wJ7ANOA70fEpj1XyszvZeaEzJwwbNiw9hZKkiRJA0grB4RHga0arg+vL2u0CLgsM1/IzAeB+6gNDJIkSZI6oJUDwixg24gYGRHrA1OBy3qscwm1rQdExObUdjl6oIVNkiRJkppo2YCQmS8CRwFXAfOBCzPz7og4LiL2r692FbA0Iu4BrgWOycylrWqSJEmS1FzLTnMKkJlXAFf0WPblhssJfLr+IUmSJKnDOn2QsiRJkqSCOCBIkiRJqjggSJIkSao4IEiSJEmqOCBIkiRJqjggSJIkSao4IEiSJEmqOCBIkiRJqjggSJIkSao4IEiSJEmqOCBIkiRJqjggSJIkSao4IEiSJEmqOCBIkiRJqjggSJIkSaqs29cVI+J1wOAV1zPz4ZYUSZIkSeqYXrcgRMT+EfEb4EHgv4GFwH+1uEuSJElSB/RlF6OvAzsD92XmSGBP4JaWVkmSJEnqiL4MCC9k5lJgnYhYJzOvBSa0uEuSJElSB/TlGIQnImJjYCZwTkT8L/B0a7MkSZIkdUJftiAcADwD/BNwJfBbYL9WRkmSJEnqjL4MCF/OzJcz88XMPCszTwY+3+owSZIkSe3XlwFh79Use1d/h0iSJEnqvDUegxARRwBHAttExLyGm4YAN7Y6TJIkSVL7NTtI+Vxq73fwr8CxDcufysw/tLRKkiRJUkescRejzHwyMxdm5jRgK+CvM/Mhaqc7Hdm2QkmSJElt05d3Uv4KtYOS/2990frAj1sZJUmSJKkz+nKQ8oHA/tTf+yAzf0ftOARJkiRJa5m+DAjPZ2YCCRARG7U2SZIkSVKn9GVAuDAivgtsGhEfB34JfL+1WZIkSZI6odlZjADIzG9FxN7AH4HtqL1x2i9aXiZJkiSp7XodEOruBF5DbTejO1uXI0mSJKmT+nIWo48BvwbeC0wBbomIw1odJkmSJKn9+rIF4Rhgx8xcChARQ4GbgP9sZZgkSZKk9uvLQcpLgacarj9VXyZJkiRpLbPGLQgR8en6xfuBWyPiUmrHIBwAzGtDmyRJkqQ2a7YFYUdqb4j2t8Al1N8HAbgUeLC1WZIkSZI6odkxCOOBzwPvA05pT44kSZKkTmo2IHwXuAYYCcxuWB7UtiZs08IuSZIkSR2wxl2MMvPkzBwFnJGZ2zR8jMxMhwNJkiRpLdTrWYwy84h2hEiSJEnqvL6c5lSSJEnSAOGAIEmSJKnigCBJkiSp4oAgSZIkqeKAIEmSJKnigCBJkiSp0tIBISL2jYgFEXF/RBzbZL33RURGxIRW9kiSJElqrmUDQkQMAk4F3gWMBqZFxOjVrDcE+Afg1la1SJIkSeqbVm5BmATcn5kPZObzwPnAAatZ7+vAdODZFrZIkiRJ6oNWDghvBB5puL6ovqwSEeOBrTLz583uKCIOj4jZETF78eLF/V8qSZIkCejgQcoRsQ5wIvCZ3tbNzO9l5oTMnDBs2LDWx0mSJEkDVCsHhEeBrRquD68vW2EIMAa4LiIWAjsDl3mgsiRJktQ5rRwQZgHbRsTIiFgfmApctuLGzHwyMzfPzBGZOQK4Bdg/M2e3sEmSJElSEy0bEDLzReAo4CpgPnBhZt4dEcdFxP6telxJkiRJr966rbzzzLwCuKLHsi+vYd09WtkiSZIkqXe+k7IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkSksHhIjYNyIWRMT9EXHsam7/dETcExHzIuKaiNi6lT2SJEmSmmvZgBARg4BTgXcBo4FpETG6x2pzgAmZOQ64CPhmq3okSZIk9a6VWxAmAfdn5gOZ+TxwPnBA4wqZeW1mPlO/egswvIU9kiRJknrRygHhjcAjDdcX1ZetyUeB/1rdDRFxeETMjojZixcv7sdESZIkSY2KOEg5Ig4BJgD/trrbM/N7mTkhMycMGzasvXGSJEnSALJuC+/7UWCrhuvD68tWEhF7AV8Eds/M51rYI0mSJKkXrdyCMAvYNiJGRsT6wFTgssYVImJH4LvA/pn5vy1skSRJktQHLRsQMvNF4CjgKmA+cGFm3h0Rx0XE/vXV/g3YGJgREXMj4rI13J0kSZKkNmjlLkZk5hXAFT2Wfbnh8l6tfHxJkiRJr0wRBylLkiRJKoMDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqSKA4IkSZKkigOCJEmSpIoDgiRJkqRKSweEiNg3IhZExP0Rcexqbt8gIi6o335rRIxoZY8kSZKk5lo2IETEIOBU4F3AaGBaRIzusdpHgccz86+AbwPTW9UjSZIkqXet3IIwCbg/Mx/IzOeB84EDeqxzAHBW/fJFwJ4RES1skiRJktTEui287zcCjzRcXwTstKZ1MvPFiHgSGAosaVwpIg4HDq9fXRYRC/7cuIDNez7Oqiu1b1bpUw+U12RPMT1QXpM9zZXWA+U12dOc/3f0zp7eldZkT3P92LP1mm5o5YDQbzLze8D3+vM+I2J2Zk7oz/v8c5TWA+U12dO70prsaa60HiivyZ7mSuuB8prs6V1pTfY0146eVu5i9CiwVcP14fVlq10nItYFNgGWtrBJkiRJUhOtHBBmAdtGxMiIWB+YClzWY53LgA/XL08BfpWZ2cImSZIkSU20bBej+jEFRwFXAYOA/8zMuyPiOGB2Zl4G/BA4OyLuB/5AbYhol37dZakflNYD5TXZ07vSmuxprrQeKK/JnuZK64HymuzpXWlN9jTX8p7wF/aSJEmSVvCdlCVJkiRVHBAkSZIkVRwQJEmSJFUcECRJkiRVHBCAiDi0Q4/7lojYMyI27rF83w71TIqIifXLoyPi0xHx7k60rE5E/KjTDY0iYtf6c/TODj3+ThHx2vrl10TE1yLi8oiYHhGbdKDn6IjYqvc12yMi1o+ID0XEXvXrH4yI/4iIT0bEeh3s2iYiPhsRJ0XEiRHxiRWvo7Q2iIjXdbqhdBExtNMNUjMOCDVfa/cDRsTRwKXAp4C7IuKAhpv/pQM9XwFOBk6PiH8F/gPYCDg2Ir7YgZ7LenxcDrx3xfV299Sbft1w+ePUnqMhwFci4tgOJP0n8Ez98knU3mhwen3ZGR3o+Tpwa0RcHxFHRsSwDjQ0OgP4G+AfIuJs4P3ArcBE4AedCKr/u/8OMLjesQG1N4u8JSL26EDPJhFxQkTcGxF/iIilETG/vmzTdvc0ExH/1aHHfW1E/GtEnB0RH+xx22kd6NkyIk6PiFMjYmhEfDUi7oyICyPi9R3o+YseH0OBX0fEZhHxFx3o2bfh8iYR8cOImBcR50bEFu3uqXecEBGb1y9PiIgHqH2vfCgidu9Az+0R8aWI+Mt2P/bq1J+TayPixxGxVUT8IiKejIhZEbFjB3o2jojjIuLuesfiiLglIj7S7paGprZ/rx4wpzmNiHlrugnYLjM3aHPPncAumbksIkYAFwFnZ+ZJETEnM9v6j6LeswO1H1h+DwzPzD9GxGuAWzNzXJt7bgfuofaDXFJ7nc6j/l4Zmfnf7eypN1WvS0TMAt6dmYsjYiPglswc2+ae+Zk5qn759swc33Db3Mzcoc09c4C3AnsBHwD2B26j9rr9JDOfanPPvMwcF7V3aX8UeENmvhQRAdzR7q/petOdwA71jg2BKzJzj4h4E3BpB/7dXwX8CjgrM39fX7YltTew3DMz27p1LCLGr+km4GeZ2YkfgC8GfgPcAhwGvAB8MDOf6/nvrk09VwI/p/YLnA8C5wDnAu8B9srMA9b8t1vS8zLwUI/Fw4FFQGbmNm3uqV6TiPgBtf/Pvg+8F9g9M9/Tzp56x50r/n+IiGuBz2XmrIjYDjg3Mye0uedB4GLgIGrPz3nABZn5u3Z2NPT8GvgKsCnwTeCfMvOiiNgTOD4zd2lzz6XAT4FfUnuONgLOB74EPJqZX2hnT72p/d+rM3NAfACPUfsBeOseHyOA33Wg5+4e1zcGrgROBOZ2oGfO6i7Xr3eiZx3gn4BfUPuBCuCBDn8N3QFsBgyl9mZ/q33+2tgzAzi0fvkMYEL98nbArA703N7j+nrUhoTzgMUd6LkLWL/+mj0F/EV9+WBgfoe+hu4ENqhf3qzx6wi4qwM9C17NbS3seYnaf4LXruZjeYdes7k9rn8RuLH+feD2DvTMabj8cLPWNvV8pv5/19iGZQ924rWqP/btDZd7vnZtf37qjzsfWLd++ZYet93Z4efo7cBp1AaFa4HDO9Azp+Fyz6/pOe1o6PGYd/S4Pqv+5zrAve3uqT92279Xt+ydlAv0M2DjzJzb84aIuK7tNfBYROywoidrWxL2o7bbSFt/E133fERsmJnPUPstMFDbrAW83O6YzHwZ+HZEzKj/+RgtfOfvPtqE2m/EA8iIeH1m/k/UjiGJDvR8DDgpIr4ELAFujohHgEfqt7XbSs9BZr4AXAZcVv9tebv9ELiX2ju5fxGYUd+0vzO13wZ1wg+AWRFxK7X/mKcDRG13rD90oOehiPgctd9KPVZv2QL4CLWvo3abD/x9Zv6m5w31r+1O2CAi1ql/TyIzvxERjwIzqf1ip90adw3ueVxW23cbzsz/FxEXUPs+/Qi13wR3cteE10XEp6l9P3ptRETWf4qic7tVnwZcEREnAFdGxEnAT4C/BuZ2qAmAzLweuD4iPgXsTW3rb7vfNfjZqB3Ltwm1/1vfk5mX1He/eqnNLQBPR8SumXlDROxP/XtzZr5c3wLdCW3/Xj1gdjHqq4jYLDMfb8PjDAdezPqmoh63Tc7MG9vcs0FmPrea5ZsDr8/MO9vZs5qOvwEmZ49Ne53q6dGwIbBFZj7YiaaoHeA6ktoAtWjFN4+G29v1NbRdZt7Xh/Xa9vxExBsAMvN39f0096L2G6rG40na/Xr9H2AUtS0G965hnXa9ZpsBxwIHACsOLH2M2mA3PTPbOrRExBRqv1FdsJrb3pOZl7Szp/643wSuzsxf9li+L3BKZm7b5p7jgG9m5rIey/8KOCEzp7Szp0fD/sAXgBGZuWWHGr7SY9FpWdsVdEtqz9uHOtS1B3AEtS2861L7oe4S4Iz6L1Pa2XJ+Zk5t52M2ExHbU9u16GVqew4cQW3XmUepbdG4sc0946j9Mmdb4G7gsMy8r/6LnGmZeXI7e+pNbf9e7YDQQyf2KW3GnuZK64HymuxprrQeKLNJ6ov6cWt/mZl3dbpF0qvnWYxW1anNR2tiT3Ol9UB5TfY0V1oPFNAUHTr985qU1gPlNZXQk5nLVwwHJfQ0Kq0Hymuyp7lO9sSfTo2/UY/lLTk1vgPCqkrbpGJPc6X1QHlN9jRXWg+U0dT20z/3orQeKK/JnuZK64HymuxpriM9sfKp8e+ONpwav9MHfUqSOiSan/657eeML60Hymuyp7nSeqC8JnuaK62n7uPAW7Ph1PgRMSIzT6JFW5wdEFbV8U37PdjTXGk9UF6TPc2V1gPta9oC2AfoeUB0ADe1qaFRaT1QXpM9zZXWA+U12dNcaT0A66w4MUFmLqwf9H5RRGyNA8KfJ3p5R8eGI8D3bEOOPb0orQfKa7KnudJ6oMim0k7/XFoPlNdkT3Ol9UB5TfY0V1oPdODU+APmLEZRe+fAFe/I21Nm+9/t0Z4u6oHymuzprh4os6kvSjidcKPSeqC8JnuaK60Hymuyp7k2n7K77afGHzADgiTp1SnttKul9UB5TfY0V1oPlNdkT3Ol9UD/Ng24sxhFzSER8c/162+KiEn22NOtTfZ0V0+pTb0o7TiN0nqgvCZ7miutB8prsqe50nqgH5sG3IBA7S3PdwE+WL/+FHBq53Ls6UVpPVBekz3NldYDZTY1U9qm5tJ6oLwme5orrQfKa7KnudJ6oB+bBsxByg12yszxETEHIDMfj4j17bGni5vs6a6eUpskSQIG5haEFyJiEPUpKyKGAS/bY88rUFqTPd3VA2U2NVPapvTSeqC8JnuaK60Hymuyp7nSeqAfmwbiFoSTgZ8Cr4uIbwBTgC/ZY88rUFqTPd3VA4U0RWGnXS2tB8prsqe50nqgvCZ7miutBzrTNCDPYhQRb6H2JAZwTWbOt8eeV6K0Jnu6qwfKaIrCTrtaWg+U12RPd/VAeU32dFcPdKZpwAwIr2D6agt7miutB8prsqe50nqgzCZJknoaSANC4/T1JmpvoR3ApsDDmTnSHnu6qcme7uoptaneFcDBwMjM/HpEvAnYMjN/bU+ZTfZ0V0+JTfZ0V0+7mwbMQcqZObK+CeaXwN9m5uaZORTYD7jaHnu6rcme7uoptamutNOultYD5TXZ01xpPVBekz3NldYD7WzKzAH1AdzZl2X22NMtTfZ0V0+JTcDt9T/nNCy7w55ym+zprp4Sm+zprp52Nw3Esxj9LiK+BPy4fv1g4Hf22PMKlNZkT3f1QHlNpZ12tbQeKK/Jnu7qgfKa7OmuHmhj04DZxajBNGAYtVMM/hR4XX2ZPfb0VWlN9nRXD5TX1PO0qzcA/2LPSkprsqe7eqC8Jnu6qwfa2DRgDlLuKSKGUDs11LJOt4A9vSmtB8prsqe50nqgrKYSTrtacg+U12RPd/VAeU32dFcPtK9pwA0IETEW+BGw4nSDS4APZ+Zd9tjTjU32dFdPSU2lnXa1tB4or8me5krrgfKa7GmutB7oUFMnD7boxAdwE/COhut7ADfZY0+3NtnTXT0lNQEPAg/U/3yJ2qCytH75wYHeU2KTPd3VU2KTPd3V06mmgXgMwkaZee2KK5l5HbBR53Ls6UVpPVBekz3NldYDhTRlYaddLa2nxCZ7uqunxCZ7uqunU00DcRejnwK3A2fXFx0CvDUzD7THnm5ssqe7ekpsiog7M3Nsb8sGak+JTfZ0V0+JTfZ0V0+7mwbiFoTDqJ095OL6x+bAofbY8wqU1mRPd/VAeU2/i4gvRcSI+scXKeDUtAX1lNhkT3f1lNhkT3f1tLVpIA4IfwlsRe1zX5/akeAz7bHnFSityZ7u6oHymko77WppPVBekz3d1QPlNdnTXT3QxqaBuIvRAuCzwF00vLlEZj5kjz3d2GRPd/WU2gRlnXYVyuuB8prsaa60HiivyZ7mSuuB9jQNxHdSXpyZl3c6ooE9zZXWA+U12dNcaT1QWFPP065GRFGnpu10T4lN9nRXT4lN9nRXT7ubBuIWhD2pbY65BnhuxfLM/Ik99nRjkz3d1VNiU0TcBHwx62dWiog9gH/JzLfZU2aTPd3VU2KTPd3V0+6mgbgF4VDgLcB6/GnTfgKd+mHBnu7qgfKa7OmuHiivaZXTrkZEUaem7XAPlNdkT3f1QHlN9nRXD7SxaSAOCBMz882djmhgT3Ol9UB5TfY0V1oPlNf0QET8MyufdvUBe1ZSWpM93dUD5TXZ01090MamgXgWo5siYnSnIxrY01xpPVBekz3NldYD5TWVdtrV0nqgvCZ7uqsHymuyp7t6oI1NA/EYhPnUTjH4ILV9f4PakeDj7LGnG5vs6a6eEpsiYgLwRWAEf9qybE/BTfZ0V0+JTfZ0V0+7mwbigLD16pZn507BaE8TpfVAeU32NFdaD5TXVNppV0vrKbHJnu7qKbHJnu7qaXfTgBsQJEkri4gbMnPXTnesUFoPlNdkT3Ol9UB5TfY0V1oPtLfJAUGSBrgCT7taVE+JTfZ0V0+JTfZ0V0+7mwbiWYwkSSsr7bSrpfVAeU32dFcPlNdkT3f1QBub3IIgSQNcRCwo6bSrpfVAeU32NFdaD5TXZE9zpfVAe5sG4mlOJUkrK+20q6X1QHlN9jRXWg+U12RPc6X1QBub3IIgSQNcgaddLaqnxCZ7uqunxCZ7uqun3U0OCJI0wBV42tWieqC8JnuaK60Hymuyp7nSeqC9TQ4IkiRJkioegyBJkiSp4oAgSZIkqeKAIEmSJKnigCBJkiSp8v8BD5S3fyzPYREAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 936x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PLOT SIMPLE MODEL AND ThresholdClassifier SEARCH RESULTS ###\n",
    "\n",
    "pd.DataFrame(\n",
    "    {'baseModel': base_model_test_fbeta,\n",
    "     'thresholdModel': threshold_model_test_fbeta},\n",
    "    index=[f\"model_{m+1}\" for m in range(len(base_model.cv_results_['params']))]\n",
    ").plot.bar(color=['blue','red'], figsize=(13,5))\n",
    "plt.ylabel('fbeta')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "25b9dae7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-19T19:36:49.020139Z",
     "iopub.status.busy": "2023-02-19T19:36:49.019326Z",
     "iopub.status.idle": "2023-02-19T19:36:49.373455Z",
     "shell.execute_reply": "2023-02-19T19:36:49.372096Z"
    },
    "papermill": {
     "duration": 0.362978,
     "end_time": "2023-02-19T19:36:49.375713",
     "exception": false,
     "start_time": "2023-02-19T19:36:49.012735",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7YAAAE9CAYAAAA/Ev6qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4yElEQVR4nO3de5wcdZXw/89JQoijXHwS1ltIBl3RZCFEGBDwAoLK5fHGGoE8AwJe4nLRZVdZYVFX0azioutPBJfoCggjgrirrOIV4UEQNEFCuAmiJCHqauABJAZWCOf3R1VCzzUzmanprp7P+/XqV1dXVVedqvp2nT5dl47MRJIkSZKkuprU7AAkSZIkSRoNC1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7CVJEmSJNXalGYHMFZmzJiRnZ2dzQ5DktQmbrrppvszc4dmx1Fn5mZJ0lgaKje3TWHb2dnJsmXLmh2GJKlNRMSqZsdQd+ZmSdJYGio3eyqyJEmSJKnWLGwlSZIkSbVmYStJkiRJqrW2ucZWkprl8ccfZ82aNTz22GPNDkVbYNq0acycOZOtttqq2aFIksaIubnetiQ3W9hK0iitWbOGbbbZhs7OTiKi2eFoBDKTBx54gDVr1rDTTjs1OxxJ0hgxN9fXluZmT0WWpFF67LHHmD59uomzhiKC6dOn+4u+JLUZc3N9bWlutrCVpDFg4qwvt50ktSf37/W1JdvOwlYaDz090NkJkyYVzz09zZl/BEyZUjyPVxzNXvYJYuXKleyyyy6VTPuaa64hIvjiF7+4qd/y5cuJCM4666xhT2c4MVa5HFJbaod9bFXL0MzcJ2FuHm8WtlLVenpg0SJYtQoyi+dFi8YvsTbOH2DDhuJ5POJo9rJrzOyyyy5cdtllm15fcskl7Lbbbk2MSFJb7GOrWoZm5j5pnJibe7Owlap2+umwfn3vfuvXF/2bNf/xiqPZy96iqjo48cQTT9Dd3c2cOXNYsGAB69ev54wzzmDPPfdkl112YdGiRWQmAJ/97GeZO3cu8+bN48gjjwTgT3/6E29729vYa6+9eMlLXsI3v/nNTdOePXs2jz32GL///e/JTL773e9yyCGHbBq+fPly9t57b+bNm8dhhx3Ggw8+CMBNN93Ebrvtxm677cY555yzafwNGzZwyimnsOeeezJv3jzOO++8sVkJ0kTSDvvYqpahmblPtWRurn9utrCVqrZ69cj6j9f8xyOOZi97C6ryAMtdd93FCSecwJ133sm2227Lueeey0knncTSpUu57bbbePTRR/nWt74FwCc+8QluvvlmVqxYwb/9278BsHjxYg444AB+9rOfcfXVV3PKKafwpz/9adP0FyxYwNe+9jV+8pOfsPvuu7P11ltvGvbWt76VM888kxUrVrDrrrvykY98BIDjjjuOs88+m1tuuaVXrP/+7//Odtttx9KlS1m6dClf+MIXuPfee0e/EqSJpB32sVUtQzNzn2rH3Fyoe262sJWqNmvWyPqP1/zHI45mL3sLqvIAy4477sjLXvYyAI466iiuu+46rr76al760pey66678qMf/Yjbb78dgHnz5tHd3c3FF1/MlCnFP799//vf5xOf+ATz589n//3357HHHmN1w5e/ww8/nK997WtccsklLFy4cFP/hx9+mIceeoj99tsPgGOOOYZrr72Whx56iIceeohXvvKVABx99NGb3vP973+fL3/5y8yfP5+XvvSlPPDAA/zyl78c/UqQJpJ22MdWtQzNzH2qHXMzm+ZV59xsYStVbfFi6Ojo3a+jo+jfrPmPVxzNXvYWVOUBlr53EIwITjjhBC6//HJuvfVW3vnOd266df63v/1tTjzxRH7+85+z55578sQTT5CZfP3rX2f58uUsX76c1atXM2fOnE3Te/azn81WW23FD37wAw488MBRxZqZnH322Zvmde+99/La1752VNOUJpx22MdWtQzNzH2qHXNzoe652cJWqlp3NyxZArNnF3dknD27eN3dPf7zB5g8uXgejziavewtqMoDLKtXr+aGG24A4Ctf+Qovf/nLAZgxYwbr1q3j8ssvB+DJJ5/kvvvu41WvehVnnnkmDz/8MOvWreOggw7i7LPP3nStz80339xvHmeccQZnnnkmkze2I2C77bbjmc98Jj/+8Y8BuOiii9hvv/3Yfvvt2X777bnuuusA6Gk4p+uggw7i85//PI8//jgAd999d69TqyQNQzvsY6tahmbmPtWOublQ99w8pdkBSBNCd3dzk2gz59/sZW8xixcX1+00nvI0VgcPXvSiF3HOOefwtre9jblz53L88cfz4IMPsssuu/DsZz+bPffcEyhuDnHUUUfx8MMPk5m85z3vYfvtt+eDH/wgJ598MvPmzePJJ59kp5122nTdz0b77rvvgPO+8MIL+Zu/+RvWr1/P85//fM4//3wAzj//fN72trcREb1+9X3HO97BypUr2X333clMdthhB77xjW+MfiVIE0077GOrWoZ2WDcaF+bmQt1zc2ys/uuuq6srly1b1uwwJE1Ad955Z6/Tgjanp6e4bmf16uLX4MWL/e7VbANtw4i4KTO7mhRSWzA3S2oWc3P9jTQ3e8RWksaZBxEkSWot5ub68xpbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlVQPPT3Q2QmTJhXPDf+7JknSFjO/SG3BwlZS6+vpKf5gbtUqyCyeFy3yy0fpoYce4txzzwXgmmuu4XWve92Yz+PYY4/d9Cfyw7Fy5Up22WWXAYftv//+bPwLmM7OTl7xilf0Gj5//vxB3zuYxmmOZhxJE4z5RRUxN49/brawldT6Tj+997+mQ/H69NObE0+LaUyew7Vhw4aKohm5Rx55hPvuuw8o/rNOksaN+UUVMTePPwvbicbTbVRHq1ePrH+rG+PP4amnnsqvfvUr5s+fzymnnMK6detYsGABL37xi+nu7iYzgeIX2Pe///3svvvufO1rX+P73/8+++yzD7vvvjtvectbWLdu3abpzZ07l3nz5vG+971v03yuvfZa9t13X57//Odv+oU4MznllFPYZZdd2HXXXbn00kv7xffoo49y5JFHMmfOHA477DAeffTRXsMPP/zwTe+75JJLWLhw4aZhjz32GMcddxy77rorL3nJS7j66qs3O83BlkuS+mm3/NLOqv4Oa27uNbyWuTkz2+Kxxx57pDbj4oszOzoyi5NtikdHR9FfamWzZ/dutxsfs2c3O7LMzLzjjjuGP3IFn8N77703/+qv/iozM6+++urcdttt87777ssNGzbk3nvvnT/+8Y8zM3P27Nl55plnZmbm2rVr8xWveEWuW7cuMzM/8YlP5Ec+8pG8//77c+edd84nn3wyMzMffPDBzMw85phjcsGCBblhw4a8/fbb8wUveEFmZl5++eX56le/Op944on87//+79xxxx3zt7/9ba+YPvWpT+Vxxx2XmZm33HJLTp48OZcuXboppl/84he5zz77ZGbm/Pnz8/bbb9/03rPOOmvTe++8887ccccd89FHHx10moMtV2bmfvvtt2m+fQ20DYFl2QL5rc4Pc7NaXovnF5W2IHeamydebq70iG1EHBwRd0XEPRFx6gDDZ0fEVRGxIiKuiYiZZf/5EXFDRNxeDjuiyjgnDE+3UV0tXgwdHb37dXQU/etmHD6He+21FzNnzmTSpEnMnz+flStXbhp2xBHF7vTGG2/kjjvu4GUvexnz58/nwgsvZNWqVWy33XZMmzaNt7/97fzHf/wHHQ3r/U1vehOTJk1i7ty5/P73vwfguuuuY+HChUyePJlnPetZ7LfffixdurRXPNdeey1HHXUUAPPmzWPevHm9hk+fPp1nPvOZfPWrX2XOnDm95nnddddteu+LX/xiZs+ezd133z3oNAdbLj3F3Cw1aKf80s6qzp3m5rbIzVPGfIqliJgMnAO8BlgDLI2IKzLzjobRzgK+nJkXRsQBwMeBo4H1wFsz85cR8Vzgpoj4XmY+VFW8E4Kn26iuuruL59NPL9rrrFnFl46N/etkHD6HW2+99abuyZMn88QTT2x6/fSnPx0oztZ5zWtewyWXXNLv/T/72c+46qqruPzyy/nc5z7Hj370o37TLX40HTtHHHEEJ554IhdccMGopjPUcsncLPXTTvmlnVWdO83NA6pbbq7yiO1ewD2Z+evM/DPwVeCNfcaZC/yo7L564/DMvDszf1l2/xb4A7BDhbFODLNmjay/1Eq6u2HlSnjyyeK5rl86KvgcbrPNNjzyyCMjes/ee+/N9ddfzz333APAn/70J+6++27WrVvHww8/zKGHHsq//uu/cssttww5nVe84hVceumlbNiwgbVr13Lttdey11579Rrnla98JV/5ylcAuO2221ixYkW/6Rx22GH8wz/8AwcddFC/6feU1zndfffdrF69mhe96EWDTnOw5dIm5mapr3bJL+2s6u+w5ua2yM1VFrbPA+5reL2m7NfoFuCvy+7DgG0iYnrjCBGxFzAV+FXfGUTEoohYFhHL1q5dO2aBty1Pt5Gar4LP4fTp03nZy17GLrvswimnnDKs9+ywww5ccMEFLFy4kHnz5rHPPvvwi1/8gkceeYTXve51zJs3j5e//OV8+tOfHnI6hx12GPPmzWO33XbjgAMO4JOf/CTPfvaze41z/PHHs27dOubMmcOHPvQh9thjj37T2WabbXj/+9/P1KlTe/U/4YQTePLJJ9l111054ogjuOCCC9h6660HneZgy6VNzM2S6qfq77Dm5rbIzTHWh6w3TThiAXBwZr6jfH008NLMPKlhnOcCnwN2Aq4F3gzssvG0poh4DnANcExm3jjU/Lq6utL/JxyGnh5Pt5HG2J133smcOXOG/wY/hy1noG0YETdlZleTQqqEuVlSbY0wd5qb62+kubmya2yB3wA7NryeWfbbpDyV6a8BIuIZwJsbEue2wLeB0zeXODUC3d1+SKVm83Oo5jE3S6qnqnOnubn2qjwVeSnwwojYKSKmAkcCVzSOEBEzImJjDKcBXyr7TwX+k+LmFZdXGKMkSROJuVmS1JYqK2wz8wngJOB7wJ3AZZl5e0ScERFvKEfbH7grIu4GngVsPJH9cOCVwLERsbx8zK8qVkmSJgJzsySpXVV5KjKZeSVwZZ9+H2rovhzo96tvZl4MXFxlbJI0ljKTiGh2GNoCVd1rolWZmyVNFObm+tqS3Fzlqcjto6cHOjth0qTiuby9taQ2NoLP/bRp03jggQcmXIHUDjKTBx54gGnTpjU7FA3GHKzxZHtrG+bm+trS3FzpEdu20NMDixbB+vXF61WritfgBeZSuxrh537mzJmsWbMG/9qknqZNm8bMmTObHYYGYg7WeLK9tRVzc71tSW6u7O9+xltlfynQ2Vns2PqaPbv4E29J7cfPvWjPv/sZb6POzX4WNZ5sb1LLGyo3eyry5qxePbL+kurPz73UGvwsajzZ3qRas7DdnFmzRtZfUv35uZdag59FjSfbm1RrFrabs3gxdHT07tfRUfSX1J783Eutwc+ixpPtTao1C9vN6e6GJUuK6ysiiuclS7yJgNTO/NxLrcHPosaT7U2qNW8eJUnSALx51OiZmyVJY8mbR0mSJEmS2paFrSRJkiSp1ixsJUmSJEm1ZmErSZIkSao1C1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7Bt1NMDnZ0waVLx3NPT7IjUqurQVuoQoyRNdO6rm8v1/xTXhWpuSrMDaBk9PbBoEaxfX7xetap4DdDd3by41Hrq0FbqEKMkTXTuq5vL9f8U14XaQGRms2MYE13bbJPL9tijd8/DD4cTTig+pIce2v9Nxx5bPO6/H2bOhP/5n/7jzJ4NP/4xHH10/2HvfS+8/vVw113wrnf1H/6BD8CrXw3Ll8PJJ/cf/s//DPvuCz/5CfzjP/Yf/pnPwPz58MMfwsc+1n/4eefBi14E//Vf8KlP9R9+0UWw445w6aXw+c/3H3755TBjBlxwQfHo68oroaMDzj0XLrus//BrrimezzoLvvWt3sOe9jT4zneK7o9+FK66qvfw6dPh618vuk87DW64offwmTPh4ouL7pNPLtZho513hiVLiu5Fi+Duu3sPnz+/WH8ARx0Fa9b0Hr7PPvDxjxfdb34zPPBA7+EHHggf/GDRfcgh8OijTw278caB28rWW8PeexfdI2l7Cxb0H3788XDEEXDffVvW9u68E/7wh8FjtO0V3XVrewCvex28731F9/7700+z214b7ffixBNvysyu/iNpuEadm9u9jR5zzODfPT78YfePVe8fB8vnM2fCC17Qv387tb2++8fhfLex7RXd5ub+w1skN3sq8kYDfZgBVq8e3zjU+gZrK4P1b4aBilporRglaaLzu0dzDbb++xY9E0EdvttIm9E+R2y7unLZsmVbPoHOzuK0i75mz4aVK7d8umo/dWgrdYhRanER4RHbURp1bm537quby/X/FNeFamKo3OwR240WLy4OcTfq6Cj6S43q0FbqEKMkTXTuq5vL9f8U14XagIXtRt3dxbn5s2dDRPG8ZIkXzKu/OrSVOsQoSROd++rmcv0/xXWhNuCpyJIkDcBTkUfP3CxJGkueiixJkiRJalsWtpIkSZKkWrOwlSRJkiTVmoVtO+rpKW7bPmlS8dzT0+yIxl+7roNWXK6BYqoqzlZc/omg73o/4YTW3A62D0lVGut9zMbpRcCUKcVzFfsu9431MJzt1DjOjBnFY3Pbtdnbfzznn5lt8dhjjz1SmXnxxZkdHZnw1KOjo+g/UbTrOmjF5Roopq22ypw6dezjbMXlnwgGWu99H62wHSpoH8CybIH8VueHuVltY6z3MUPtW8dyn2rurIfhbKfN5eOBtmuzt/8452bvitxu/IPt9l0Hrbhcg8U0kNHG2YrLPxEMdxs3eztU0D68K/LomZvVNsZ6H7O5fetY7VPNnfUwnO00nHzcd7s2e/uPc262sG03kyYVv4f0FQFPPjn+8TRDu66DVlyuwWIayGjjbMXlnwiGu42bvR0qaB8WtqNnblbbGOt9zOb2rWO1TzV31sNwttNw8nHf7drs7T/OubnSa2wj4uCIuCsi7omIUwcYPjsiroqIFRFxTUTMbBh2TET8snwcU2WcbWXWrJH1b0ftug5acblGMu/RxtmKyz8RDHf9Nns72D6GzdwsbYGx3sds7n1jte9y31gPw9lOw9lmfcdp9vYf5/lXVthGxGTgHOAQYC6wMCLm9hntLODLmTkPOAP4ePne/wX8E/BSYC/gnyLimVXF2lYWL4aOjt79OjqK/hNFu66DVlyugWLaaiuYOrV3v7GIsxWXfyIYaL331QrbwfYxLOZmaQuN9T5mqH3rWO673DfWw3C20+by8UDbtdnbf7znP9jFt6N9APsA32t4fRpwWp9xbgd2LLsD+GPZvRA4r2G884CFQ83PG1Q0uPjizNmzMyOK54l4g4B2XQetuFwDxVRVnK24/BNB3/V+/PGtuR3GuH3QhjePMjdLozDWOWjj9CBz8uTiuYp9qrmzHoaznRrHmT69eGxuuzZ7+49jbq7sGtuIWAAcnJnvKF8fDbw0M09qGOcrwE8z8/+LiL8Gvg7MAI4DpmXmx8rxPgg8mpln9ZnHImARwKxZs/ZYNdyb2EiStBnteI2tuVmSVGdNu8Z2GN4H7BcRNwP7Ab8BNgz3zZm5JDO7MrNrhx12qCpGSZImEnOzJKl2plQ47d8AOza8nln22yQzfwv8NUBEPAN4c2Y+FBG/Afbv895rKoxVkqSJwNwsSWpLVR6xXQq8MCJ2ioipwJHAFY0jRMSMiNgYw2nAl8ru7wGvjYhnljemeG3ZT5IkbTlzsySpLVVW2GbmE8BJFEnvTuCyzLw9Is6IiDeUo+0P3BURdwPPAhaX7/1/wEcpEvBS4IyynyRJ2kLmZklSu6rs5lHjzT+BlySNpXa8edR4MzdLksZSK988SpIkSZKkUbGwlSRJkiTVmoWtJEmSJKnWLGwlSZIkSbVmYStJkiRJqjULW0mSJElSrVnYSpIkSZJqzcJWkiRJklRrFraSJEmSpFqzsJUkSZIk1ZqFrSRJkiSp1ixsJUmSJEm1ZmErSZIkSao1C1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7CVJEmSJNWaha0kSZIkqdYsbCVJkiRJtWZhK0mSJEmqNQtbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlSRJkiTVmoWtJEmSJKnWLGwlSZIkSbVmYStJkiRJqjULW0mSJElSrVnYSpIkSZJqrdLCNiIOjoi7IuKeiDh1gOGzIuLqiLg5IlZExKFl/60i4sKIuDUi7oyI06qMU5KkicLcLElqR5UVthExGTgHOASYCyyMiLl9RvsAcFlmvgQ4Eji37P8WYOvM3BXYA3hXRHRWFaskSROBuVmS1K6qPGK7F3BPZv46M/8MfBV4Y59xEti27N4O+G1D/6dHxBTgacCfgT9WGKskSROBuVmS1JaqLGyfB9zX8HpN2a/Rh4GjImINcCXw7rL/5cCfgN8Bq4GzMvP/VRirJEkTgblZktSWmn3zqIXABZk5EzgUuCgiJlH8orwBeC6wE/DeiHh+3zdHxKKIWBYRy9auXTuecUuS1K7MzZKk2qmysP0NsGPD65llv0ZvBy4DyMwbgGnADOD/AN/NzMcz8w/A9UBX3xlk5pLM7MrMrh122KGCRZAkqa2YmyVJbanKwnYp8MKI2CkiplLcgOKKPuOsBg4EiIg5FMlzbdn/gLL/04G9gV9UGKskSROBuVmS1JYqK2wz8wngJOB7wJ0Ud1i8PSLOiIg3lKO9F3hnRNwCXAIcm5lJccfGZ0TE7RRJ+PzMXFFVrJIkTQTmZklSu4oiV9VfV1dXLlu2rNlhSJLaRETclJn9TrXV8JmbJUljaajc3OybR0mSJEmSNCoWtpIkSZKkWrOwlSRJkiTVmoWtJEmSJKnWLGwlSZIkSbVmYStJkiRJqjULW0mSJElSrVnYSpIkSZJqbcpwR4yIvwCmbXydmasriUiSJA2LuVmSpMJmj9hGxBsi4pfAvcD/BVYC36k4LkmSNAhzsyRJvQ3nVOSPAnsDd2fmTsCBwI2VRiVJkoZibpYkqcFwCtvHM/MBYFJETMrMq4GuiuOSJEmDMzdLktRgONfYPhQRzwCuBXoi4g/An6oNS5IkDcHcLElSg+EcsX0jsB74O+C7wK+A11UZlCRJGpK5WZKkBsMpbD+UmU9m5hOZeWFmfhZ4f9WBSZKkQZmbJUlqMJzC9jUD9DtkrAORJEnDZm6WJKnBoNfYRsTxwAnA8yNiRcOgbYDrqw5MkiT1Zm6WJGlgQ9086isU/4n3ceDUhv6PZOb/qzQqSZI0EHOzJEkDGPRU5Mx8ODNXZuZCYEfggMxcRfHXAjuNW4SSJAkwN0uSNJjNXmMbEf9EcUOK08peU4GLqwxKkiQNztwsSVJvw7l51GHAGyj/Hy8zf0txLY8kSWoOc7MkSQ2GU9j+OTMTSICIeHq1IUmSpM0wN0uS1GA4he1lEXEesH1EvBP4IfCFasOSJElDMDdLktRgqLsiA5CZZ0XEa4A/AjtT/Cn8DyqPTJIkDcjcLElSb5stbEu3Ak+jOOXp1urCkSRJw2RuliSpNJy7Ir8D+Bnw18AC4MaIeFvVgUmSpIGZmyVJ6m04R2xPAV6SmQ8ARMR04CfAl6oMTJIkDcrcLElSg+HcPOoB4JGG14+U/SRJUnOYmyVJajDoEduI+Puy8x7gpxHxTYrreN4IrBiH2CRJUgNzsyRJAxvqiO1LKP7s/fXANyj/Kw/4JnBvtWFJkqQBmJslSRrAUNfY7g68H3gzcPb4hCNJkoZgbpYkaQBDHbE9D7iK4v/xljU8biqfNysiDo6IuyLinog4dYDhsyLi6oi4OSJWRMShDcPmRcQNEXF7RNwaEdNGsmCSJLUhc7MkSQMY9IhtZn4W+GxEfD4zjx/phCNiMnAO8BpgDbA0Iq7IzDsaRvsAcFlmfj4i5gJXAp0RMQW4GDg6M28p7/b4+EhjkCSpnZibJUka2GbvirwlibO0F3BPZv46M/8MfJXi5ha9Jg9sW3ZvB/y27H4tsCIzbyljeCAzN2xhHJIktRVzsyRJvQ3n73621POA+xperyn7NfowcFRErKH4RfjdZf+dgYyI70XEzyPiHyqMU5KkicLcLElqS1UWtsOxELggM2cChwIXRcQkilOkXw50l8+HRcSBfd8cEYsiYllELFu7du14xi1JUrsyN0uSaqfKwvY3wI4Nr2eW/Rq9HbgMIDNvAKYBMyh+Qb42M+/PzPUUvxjv3ncGmbkkM7sys2uHHXaoYBEkSWor5mZJUluqsrBdCrwwInaKiKnAkcAVfcZZDRwIEBFzKJLnWuB7wK4R0VHerGI/4A4kSdJomJslSW1pqP+xHZXMfCIiTqJIhJOBL2Xm7RFxBrAsM68A3gt8ISL+juJmFcdmZgIPRsSnKRJwAldm5rerilWSpInA3CxJaldR5Kr66+rqymXLhvUXfpIkbVZE3JSZXc2Oo87MzZKksTRUbm72zaMkSZIkSRoVC1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7CVJEmSJNWaha0kSZIkqdYsbCVJkiRJtWZhK0mSJEmqNQtbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlSRJkiTVmoWtJEmSJKnWLGwlSZIkSbVmYStJkiRJqjULW0mSJElSrVnYSpIkSZJqzcJWkiRJklRrFraSJEmSpFqzsJUkSZIk1ZqFrSRJkiSp1ixsJUmSJEm1ZmErSZIkSao1C1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7CVJEmSJNWaha0kSZIkqdYqLWwj4uCIuCsi7omIUwcYPisiro6ImyNiRUQcOsDwdRHxvirjlNpRTw90dsKkScVzT8/A/SRNLObm1jLa/bL7dUkqTKlqwhExGTgHeA2wBlgaEVdk5h0No30AuCwzPx8Rc4Ergc6G4Z8GvlNVjFK76umBRYtg/fri9apVcNxxEAF//vNT/RYtKrq7u5sTp6TxZW5uLQPtq0eyXx7t+yWpnVR5xHYv4J7M/HVm/hn4KvDGPuMksG3ZvR3w240DIuJNwL3A7RXGKLWl009/6ovORo8//lRRu9H69cW4kiYMc3MLGWhfPZL98mjfL0ntpMrC9nnAfQ2v15T9Gn0YOCoi1lD8IvxugIh4BvB+4CMVxie1rdWrqxlXUu2Zm1vIYPvf4e6XR/t+SWonzb551ELggsycCRwKXBQRkyiS6r9m5rqh3hwRiyJiWUQsW7t2bfXRSjUxa1Y140qaEMzN42Sw/e9w98ujfb8ktZMqC9vfADs2vJ5Z9mv0duAygMy8AZgGzABeCnwyIlYCJwP/GBEn9Z1BZi7JzK7M7Nphhx3GfAGkulq8GDo6evfbaiuYOrV3v46OYlxJE4a5uYUMtK8eyX55tO+XpHZSZWG7FHhhROwUEVOBI4Er+oyzGjgQICLmUCTPtZn5iszszMxO4DPAP2fm5yqMVWor3d2wZAnMnl3cMGr2bDj/fPjSl3r3W7LEG4xIE4y5uYUMtK8eyX55tO+XpHZS2V2RM/OJ8pfc7wGTgS9l5u0RcQawLDOvAN4LfCEi/o7iZhXHZmZWFZM0kXR3D/zlxi880sRlbm49g+2rx+v9ktQuol1yVVdXVy5btqzZYUiS2kRE3JSZXc2Oo87MzZKksTRUbm72zaMkSZIkSRoVC1tJkiRJUq1Z2EqSJEmSas3CVpIkSZJUaxa2kiRJkqRas7CVJEmSJNWaha0kSZIkqdYsbNtQTw90dsKkScVzT8/whkmSJGl8+J1sZFxf2pwpzQ5AY6unBxYtgvXri9erVhWvNxpsWHf3+MYpSZI0UQ31fc3vZP25vjQckZnNjmFMdHV15bJly5odRtN1dhYf9r5mzy6eBxu2cmWVUUlS/UTETZnZ1ew46szcLA1sqO9rfifrz/WljYbKzR6xbTOrV4+s/+aGSZIkaWxtyfe1icz1peHwGts2M2vW4P2HGqaRGel1Hs2+LqSZ82/2skuSWltVeWLjdCNgypTiuVXuPeJ3spFxfWk4PGLbZhYv7n0NAkBHR9Efhh6m4RnpdR7Nvi6kmfNv9rJLklpbVXmi73Q3bOg/fWhejtrc9zX15vrScHiNbRvq6YHTTy9Oz5g1q/jQb9xBDzVMwzPS6zyafV1IM+ff7GWXRsNrbEfP3KzNqSpPDDbdxulDc3OU38lGxvUlGDo3W9hKIzRpEgz0sYmAJ58c/fhjrZnzb/ayS6NhYTt65mZtTlV5YrDpNk4fzFFS3QyVm73GVhqhkV7n0ezrQpo5/2YvuySptVWVJzb3fu89IrUfC1tphBYvLq7raDTUdR4jHX+sNXP+zV52SVJrqypPDDTdvtM3R0ntxcJWGqHubliypLgGJ6J4XrJk8Os8Rjp+s+Ntl3lLklpfVXmicboAkycXz43TN0dJ7cVrbCVJGoDX2I6euVmSNJa8xlaSJEmS1LYsbCVJkiRJtWZhK0mSJEmqNQtbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlTTmenqgsxMmTSqee3qaHdHItcMyTDRuM0mj1Yz9iPsutbPxbN9Tqpu0pImopwcWLYL164vXq1YVrwG6u5sX10i0wzJMNG4zSaPVjP2I+y61s/Fu35GZYz/VJthmm67cY49lvfodfjiccEKxMg89tP97jj22eNx/PyxY0H/48cfDEUfAfffB0Uf3H/7e98LrXw933QXvelf/4R/4ALz61bB8OZx8cv/h//zPsO++8JOfwD/+Y//hn/kMzJ8PP/whfOxj/Yefdx686EXwX/8Fn/pU/+EXXQQ77giXXgqf/3z/4ZdfDjNmwAUXFI++rrwSOjrg3HPhssv6D7/mmuL5rLPgW9/qPexpT4PvfKfo/uhH4aqreg+fPh2+/vWi+7TT4IYbeg+fORMuvrjoPvnkYh022nlnWLKk6F60CO6+u/fw+fOL9Qdw1FGwZk3v4fvsAx//eNH95jfDAw/0Hn7ggfDBDxbdhxwCjz7ae/jrXgfve1/Rvf/+9DOR296NN8L//E//92y9Ney9d9Hd6m3vuuuKne9Ay/DWt9r2WrHtDafdwcja3oknxk2Z2dV/LA2Xubn/cHNz0d2K+8fbbus/T3hqP1JF2xvuvgtse+3c9tp1vzfeudlTkSWNqYF2YEP1b0WrVw/cv07LMNG0Q7uT1FwDFbVQ7X7EfZfa2Xi377Y5YtvV1ZXLli3b/IiSKtXZOfDRztmzYeXK8Y5my7TDMkw0VWyzCI/Yjpa5WXXSjH2/+UbtbLxzc6VHbCPi4Ii4KyLuiYhTBxg+KyKujoibI2JFRBxa9n9NRNwUEbeWzwdUGaeksbN4cXG6SKOOjqJ/XbTDMkw0brPhMzdLA2vGfsR9l9rZeLfvygrbiJgMnAMcAswFFkbE3D6jfQC4LDNfAhwJnFv2vx94fWbuChwDXFRVnJLGVnd3cZ3L7NkQUTwvWVKvm2C0wzJMNG6z4TE3S4Nrxn7EfZfa2Xi378pORY6IfYAPZ+ZB5evTADLz4w3jnAf8OjPPLMf/VGbu22c6ATwAPCczBz0j29OdJEljqR1PRTY3S5LqrFmnIj8PuK/h9ZqyX6MPA0dFxBrgSuDdA0znzcDPh0qcVfP/xaTBjdfno6enuGNeRPGYMcPP4njpu41POMF9Yo3VLjebg1U125hawXDaYeM4M2YUj8212wnVvjOzkgewAPhiw+ujgc/1GefvgfeW3fsAdwCTGob/FfAr4AWDzGMRsAxYNmvWrKzCxRdndnRkwlOPjo6ivzTRjdfn4+KLM6dO7T0fyNxqKz+LVRtoG/d9tOs+EViWFeXIZj3qlpvNwaqabUytYDjtcHP5eKB2247te6jc3OxTkW8HDs7M+8rXvwb2zsw/RMRM4EfAcZl5/ebmV9XpTt6tThrceH0+BptPFfNSb0Ot+0btuB0m8KnILZObzcGqmm1MrWA47XA4+bhvu23H9t2sU5GXAi+MiJ0iYirFDSiu6DPOauDAMsg5wDRgbURsD3wbOHU4ibNKg/2f5WD9pYlkvD4fQ03Pz2K1hrt+3Q61UavcbA5W1WxjagXDaYfDaZN9x5lo7buywjYznwBOAr4H3Elxh8XbI+KMiHhDOdp7gXdGxC3AJcCx5SHmk4C/BD4UEcvLx19UFetQZs0aWX9pIhmvz8dQ0/OzWK3hrl+3Qz3ULTebg1U125hawXDa4XDaZN9xJlr7rvR/bDPzyszcOTNfkJmLy34fyswryu47MvNlmblbZs7PzO+X/T+WmU8v+218/KHKWAfj/4tJgxuvz8fixTB1av/+W23lZ7FqA23jvtwn1kudcrM5WFWzjakVDKcdbi4fD9RuJ1r7rrSwbQf+v5g0uPH6fHR3w5e+BNOnP9Vv+nQ4/3w/i1UbaBsff7z7RI0Pc7CqZhtTKxhOO+w7zvTpxWOodjvR2ndlN48ab/5XniRpLLXjzaPGm7lZkjSWmnXzKEmSJEmSKmdhK0mSJEmqNQtbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlSRJkiTVmoWtJEmSJKnWLGwlSZIkSbVmYStJkiRJqjULW0mSJElSrVnYSpIkSZJqzcJWkiRJklRrFraSJEmSpFqzsJUkSZIk1ZqFrSRJkiSp1ixsJUmSJEm1ZmHboKcHOjth0qTiuaen2RFpS7XrtmzX5ZIkqdXVNQcPN+66Lp+00ZRmB9Aqenpg0SJYv754vWpV8Rqgu7t5cWnk2nVbtutySZLU6uqag4cbd12XT2oUmdnsGMZEV1dXLlu2bIvf39lZfIj7mj0bVq7c4smqCdp1W7brckmtKiJuysyuZsdRZ6PNzVKrqGsOHm7cdV0+TTxD5WZPRS6tXj2y/mpd7bot23W5JElqdXXNwcONu67LJzWysC3NmjWy/mpd7bot23W5JElqdXXNwcONu67LJzWysC0tXgwdHb37dXQU/VUv7bot23W5JElqdXXNwcONu67LJzWysC11d8OSJcW1BBHF85IlXjBfR+26Ldt1uSRJanV1zcHDjbuuyyc18uZRkiQNwJtHjZ65WZI0lrx5lCRJkiSpbVnYSpIkSZJqzcJWkiRJklRrFraSJEmSpFqzsJUkSZIk1ZqFrSRJkiSp1ixsJUmSJEm1ZmErSZIkSaq1yMxmxzAmImItsKrZcbSxGcD9zQ6ijbl+q+c6rl67rePZmblDs4OosxbOzXVuq8beHMbeHMbeHK0c+6C5uW0KW1UrIpZlZlez42hXrt/quY6r5zpWXdS5rRp7cxh7cxh7c9Q1dk9FliRJkiTVmoWtJEmSJKnWLGw1XEuaHUCbc/1Wz3VcPdex6qLObdXYm8PYm8PYm6OWsXuNrSRJkiSp1jxiK0mSJEmqNQvbCSgiDo6IuyLinog4dYDhsyLi6oi4OSJWRMShZf/uiFje8HgyIuaXw/aIiFvLaX42ImKcF6ulVLSOF0fEfRGxbpwXpyWN9TqOiI6I+HZE/CIibo+IT4z/UrWWitrxdyPilnId/1tETB7nxVKb2Vw7Lcc5PCLuKNvdVxr6fzciHoqIb/UZ/8cN7fe3EfGNGsV+YET8vIz9uoj4yxrFfkAZ+20RcWFETGml2MtccUPZb0VEHNEw/k4R8dNympdGxNQaxX5SOb2MiBlVxF1h7D3lNG+LiC9FxFY1iv3fy3y4IiIuj4hn1CX2hvd9Nlrpe2lm+phAD2Ay8Cvg+cBU4BZgbp9xlgDHl91zgZUDTGdX4FcNr38G7A0E8B3gkGYvaxuu472B5wDrmr2MzX5UsY6BDuBVZfdU4Me240ra8bblcwBfB45s9rL6qO9jmO30hcDNwDPL13/RMOxA4PXAt4aYx9eBt9YlduBuYE7ZfQJwQR1ipzjYch+wc/n6DODtrRQ7sDPwwrL7ucDvgO3L15dt3J8B/7Zx31iT2F8CdAIrgRljHXfFsR9KkU8CuKRm633bhvd/Gji1LrGX/bqAi2ih76UesZ149gLuycxfZ+afga8Cb+wzTgLblt3bAb8dYDoLy/cSEc+h+HDemEVL/zLwpgpir4sxX8cA5fr9XQXx1tGYr+PMXJ+ZV5fdfwZ+DsysIPa6qKod/7HsnEKRZL3Rg0ZjOO30ncA5mfkgQGb+YeOAzLwKeGSwiUfEtsABwDfGOG6oLvbhfC5Hq4rYpwN/zsy7y9c/AN7cSrFn5t2Z+cuy+7fAH4AdIiIo2snl5fsvpJrvQWMee/n65sxcWUG84xH7lVmiOMhSRd6uKvY/ApTt52lUkw8riT2Ks63+BfiHCmLeYha2E8/zKH4R3WhN2a/Rh4GjImINcCXw7gGmcwTFL2Mbp7lmM9OcSKpYx+qt0nUcEdtTHE24agxiravK1nFEfI8iQT7CU18EpS0xnHa6M7BzRFwfETdGxMEjmP6bgKsafpAZS1XF/g7gyvJzeTRQxWUVVcR+PzAlIrrK1wuAHcck2t7GJPaI2Ivix7lfURTlD2XmE0NMcyxUEft4qTT28hTko4HvjmnUhcpij4jzgf8GXgycPdaBU13sJwFXtNoBFwtbDWQhxalLMylO8bgoIja1lYh4KbA+M29rVoBtwHVcvS1ax1Fc03UJ8NnM/PV4BlxDW7SOM/MgitPqt6Y4yiFVaQrFqXb7U7TZL5Q/Xg3HQpr7A+OWxP53wKHl5/J8ilMcm2FEsZdH3I4E/jUifkbxw9eG6sMc0JCxl2eqXQQcl5lPNiPAIUzU2M8Frs3MH49PqP1sUeyZeRzFab53UvwQ3Awjij0ingu8hWoK8VGxsJ14fkPvX0Bnlv0avZ3iWhEy8wZgGtB4M4Ej6Z3of0PvUz8GmuZEUsU6Vm9VruMlwC8z8zNjFWxNVdqOM/Mx4Jv0PyVKGonhtNM1FEcWHs/MeymuQX3h5iZc3kRnL+DbYxRrX2Mee0TsAOyWmT8te10K7Dt2IW9SyXrPzBsy8xWZuRdwbfmesTaq2MvT078NnJ6ZN5bjPwBsH0/d7Kqq70FVxD5eKos9Iv6J4hTZv69b7ACZuYHiFOEqTr2vIvaXAH8J3BMRK4GOiLingthHzMJ24lkKvDCKu/dNpfjieUWfcVZT3NiBiJhD8WV1bfl6EnA4va+Z+x3wx4jYu7xO4K0UX1gnqjFfx+qnknUcER+juCbt5CqDr4kxX8cR8Yzyl9+NR8b/N/CLipdD7W047fQbFEciNharOwPDORtjAcXNjR4bs2h7qyL2B4HtImLn8vVrKI4EjbVK1ntE/EX5vDXwfoqbMI21LY69HP8/gS9n5qbLKMqjzVdTtBmAY6jme9CYxz6OKok9It4BHAQsrPAI9JjHHoW/3NgNvIFq8mEV7f3bmfnszOzMzE6KM7Mqufv6iGUL3MHKx/g+KE4ZvJviPPnTy35nAG8ou+cC11PcOW058NqG9+4P3DjANLuA28ppfg6IZi9nG67jT1L8qvZk+fzhZi9nO61jil8xk+JL4PLy8Y5mL2ebreNnUSTZFeX+4mxgSrOX00e9H8Nop0FxOu4dwK003Imb4u7na4FHy/3qQQ3DrgEOrlvswGHluLeUy/D8GsX+L+U++C7g5FZb78BRwOMNOWI5ML8c9nyKmxfdA3wN2LpGsb+n3A5PUNxs7Is1iv2Jcnob+3+oDrFTHFy8vhz3NqCHhrskt3LsA0y/Ze6KHGVAkiRJkiTVkqciS5IkSZJqzcJWkiRJklRrFraSJEmSpFqzsJUkSZIk1ZqFrSRJkiSp1ixsJUmSpDYUEV0R8dkhhj83Iprxv7bSmPPvfiRJkqRhiIjJmblhos5famUesZXaUER8IyJuiojbI2JR2W9dw/AFEXFB2f2siPjPiLilfOzbpLAlSRozg+TCv4mIf2kY59iI+FzZfVRE/CwilkfEeRExuey/LiI+FRG3APtExIciYmlE3BYRSyIiyvH2jIgV5fv/JSJuK/tPLl8vLYe/a4BYOyPiFxHRExF3RsTlEdFRDlsZEWdGxM+Bt0TEayPihoj4eUR8LSKe0TD/n5S5/GcRsU1E7B8R3yqH71fGtjwibi6HdzbEOS0izo+IW8vhr2pYR/8REd+NiF9GxCer2mbSaFjYSu3pbZm5B9AFvCcipg8x7meB/5uZuwG7A7ePR4CSJFVsoFz4deCwhnGOAL4aEXPK7pdl5nxgA9BdjvN04KeZuVtmXgd8LjP3zMxdgKcBryvHOx94V8P7N3o78HBm7gnsCbwzInYaIN4XAedm5hzgj8AJDcMeyMzdgR8CHwBeXb5eBvx9REwFLgX+tsznrwYe7TP99wEnlvG9YoDhJwKZmbsCC4ELI2JaOWx+uX52BY6IiB0HiF9qKgtbqT29p/xl+UZgR+CFQ4x7APB5gMzckJkPj0N8kiRVrV8uzMy1wK8jYu+y0H0xcD1wILAHsDQilpevn19OZwNFQbzRqyLipxFxK0UO/auI2B7YJjNvKMf5SsP4rwXeWk73p8B0Bs7L92Xm9WX3xcDLG4ZdWj7vDcwFri+ndwwwm6Io/l1mLgXIzD9m5hN9pn898OmIeA+w/QDDX17Ol8z8BbAK2LkcdlVmPpyZjwF3lPOUWsqUZgcgaWxFxP4Uv9Tuk5nrI+IaYBrQeEH9tP7vlCSpPQyRCwG+ChwO/AL4z8zM8nTiCzPztAEm99jG61rLI5jnAl2ZeV9EfJjN59QA3p2Z39vMeH1vfNP4+k8N0/pBZi7sNYOIXTczbTLzExHxbeBQisL4IOCxzb2v9D8N3RuwhlAL8oit1H62Ax4sE/mLKX7dBfh9RMyJiEn0Pg3rKuB42HQd0HbjG64kSWNusFwI8J/AGylOt/1q2e8qYEFE/AVARPyviBjoqOTGIvb+8trWBQCZ+RDwSES8tBx+ZMN7vgccHxFbldPeOSKePsC0Z0XEPmX3/wGuG2CcG4GXRcRfltN6ekTsDNwFPCci9iz7bxMRvYrPiHhBZt6amWcCSymOVjf6MeXp1+U0Z5XTlWrBwlZqP98FpkTEncAnKJIgwKnAt4CfAL9rGP9vKU6ruhW4ieIUJ0mS6mywXEhmPgjcCczOzJ+V/e6guHb1+xGxAvgB8Jy+Ey0L2C8At1EUrEsbBr8d+EJ5ivDTgY2X9nyR4vTdn5c3ajqPgY943gWcWMb8TMrLhPrMfy1wLHBJGecNwIsz888U18CeXZ5+/QP6H0k+ubzh1QrgceA7fYafC0wqvw9cChybmf+DVBP+3Y8kSZI0ShHxjMxcV3afCjwnM/92mO/tBL5V3pBK0hbw/HhJkiRp9P53RJxG8f16FcWRVUnjxCO2kiRJkqRa8xpbSZIkSVKtWdhKkiRJkmrNwlaSJEmSVGsWtpIkSZKkWrOwlSRJkiTVmoWtJEmSJKnW/n9EJD0tOryCIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### PLOT SIMPLE MODEL AND ThresholdClassifier SEARCH RESULTS ###\n",
    "\n",
    "plt.figure(figsize=(16,5))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.scatter(\n",
    "    base_model_test_roc_auc,\n",
    "    base_model_test_fbeta,\n",
    "    c='blue', label='baseModel'\n",
    ")\n",
    "plt.scatter(\n",
    "    threshold_model_test_roc_auc,\n",
    "    threshold_model_test_fbeta,\n",
    "    c='red', label='thresholdModel'\n",
    ")\n",
    "plt.axhline(np.median(base_model_test_fbeta), \n",
    "            color='blue', linestyle='--')\n",
    "plt.axhline(np.median(threshold_model_test_fbeta), \n",
    "            color='red', linestyle='--')\n",
    "plt.ylabel('fbeta'); plt.xlabel('auc'); plt.legend()\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(\n",
    "    base_model_test_avg_precision,\n",
    "    base_model_test_fbeta,\n",
    "    c='blue', label='baseModel'\n",
    ")\n",
    "plt.scatter(\n",
    "    threshold_model_test_avg_precision,\n",
    "    threshold_model_test_fbeta,\n",
    "    c='red', label='thresholdModel'\n",
    ")\n",
    "plt.axhline(np.median(base_model_test_fbeta), \n",
    "            color='blue', linestyle='--')\n",
    "plt.axhline(np.median(threshold_model_test_fbeta), \n",
    "            color='red', linestyle='--')\n",
    "plt.ylabel('fbeta'); plt.xlabel('average precision'); plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 273.06525,
   "end_time": "2023-02-19T19:36:52.002763",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-02-19T19:32:18.937513",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
